80 #define DEBUG_TYPE "dagcombine"
82 STATISTIC(NodesCombined ,
"Number of dag nodes combined");
83 STATISTIC(PreIndexedNodes ,
"Number of pre-indexed nodes created");
84 STATISTIC(PostIndexedNodes,
"Number of post-indexed nodes created");
85 STATISTIC(OpsNarrowed ,
"Number of load/op/store narrowed");
86 STATISTIC(LdStFP2Int ,
"Number of fp load/store pairs transformed to int");
88 STATISTIC(NumFPLogicOpsConv,
"Number of logic ops converted to fp ops");
92 cl::desc(
"Enable DAG combiner's use of IR alias analysis"));
96 cl::desc(
"Enable DAG combiner's use of TBAA"));
101 cl::desc(
"Only use DAG-combiner alias analysis in this"
109 cl::desc(
"Bypass the profitability model of load slicing"),
114 cl::desc(
"DAG combiner may split indexing from loads"));
118 cl::desc(
"DAG combiner enable merging multiple stores "
119 "into a wider store"));
123 cl::desc(
"Limit the number of operands to inline for Token Factors"));
127 cl::desc(
"Limit the number of times for the same StoreNode and RootNode "
128 "to bail out in store merging dependence check"));
132 cl::desc(
"DAG combiner enable reducing the width of load/op/store "
137 cl::desc(
"DAG combiner enable load/<replace bytes>/store with "
138 "a narrower store"));
148 bool LegalDAG =
false;
149 bool LegalOperations =
false;
150 bool LegalTypes =
false;
152 bool DisableGenericCombines;
193 void AddUsersToWorklist(
SDNode *
N) {
199 void AddToWorklistWithUsers(
SDNode *
N) {
200 AddUsersToWorklist(
N);
207 void clearAddedDanglingWorklistEntries() {
209 while (!PruningList.
empty()) {
212 recursivelyDeleteUnusedNodes(
N);
216 SDNode *getNextWorklistEntry() {
218 clearAddedDanglingWorklistEntries();
222 while (!
N && !Worklist.
empty()) {
227 bool GoodWorklistEntry = WorklistMap.
erase(
N);
228 (void)GoodWorklistEntry;
229 assert(GoodWorklistEntry &&
230 "Found a worklist entry without a corresponding map entry!");
240 : DAG(
D), TLI(
D.getTargetLoweringInfo()),
241 STI(
D.getSubtarget().getSelectionDAGInfo()), OptLevel(OL), AA(AA) {
245 MaximumLegalStoreInBits = 0;
251 VT.getSizeInBits().getKnownMinSize() >= MaximumLegalStoreInBits)
252 MaximumLegalStoreInBits = VT.getSizeInBits().getKnownMinSize();
255 void ConsiderForPruning(
SDNode *
N) {
262 void AddToWorklist(
SDNode *
N) {
264 "Deleted Node added to Worklist");
271 ConsiderForPruning(
N);
273 if (WorklistMap.
insert(std::make_pair(
N, Worklist.
size())).second)
281 StoreRootCountMap.
erase(
N);
283 auto It = WorklistMap.
find(
N);
284 if (It == WorklistMap.
end())
288 Worklist[It->second] =
nullptr;
289 WorklistMap.
erase(It);
292 void deleteAndRecombine(
SDNode *
N);
293 bool recursivelyDeleteUnusedNodes(
SDNode *
N);
301 return CombineTo(
N, &Res, 1, AddTo);
308 return CombineTo(
N, To, 2, AddTo);
314 unsigned MaximumLegalStoreInBits;
320 unsigned BitWidth =
Op.getScalarValueSizeInBits();
332 AddToWorklist(
Op.getNode());
334 CommitTargetLoweringOpt(TLO);
341 bool SimplifyDemandedVectorElts(
SDValue Op) {
343 if (
Op.getValueType().isScalableVector())
346 unsigned NumElts =
Op.getValueType().getVectorNumElements();
348 return SimplifyDemandedVectorElts(
Op, DemandedElts);
352 const APInt &DemandedElts,
353 bool AssumeSingleUse =
false);
354 bool SimplifyDemandedVectorElts(
SDValue Op,
const APInt &DemandedElts,
355 bool AssumeSingleUse =
false);
357 bool CombineToPreIndexedLoadStore(
SDNode *
N);
358 bool CombineToPostIndexedLoadStore(
SDNode *
N);
522 bool reassociationCanBreakAddressingModePattern(
unsigned Opc,
540 bool NotExtCompare =
false);
541 SDValue convertSelectOfFPConstantsToLoadOffset(
555 const SDLoc &
DL,
bool foldBooleans);
559 SDValue &CC,
bool MatchStrict =
false)
const;
560 bool isOneUseSetCC(
SDValue N)
const;
583 bool DemandHighBits =
true);
587 unsigned PosOpcode,
unsigned NegOpcode,
591 unsigned PosOpcode,
unsigned NegOpcode,
606 SDValue VecIn2,
unsigned LeftIdx,
641 int64_t OffsetFromBase;
644 : MemNode(
N), OffsetFromBase(
Offset) {}
649 StoreSource getStoreSource(
SDValue StoreVal) {
656 return StoreSource::Extract;
668 bool isMulAddWithConstProfitable(
SDNode *MulNode,
676 EVT LoadResultTy,
EVT &ExtVT);
681 EVT &MemVT,
unsigned ShAmt = 0);
689 bool BackwardsPropagateMask(
SDNode *
N);
701 EVT MemVT,
unsigned NumStores,
702 bool IsConstantSrc,
bool UseVector,
716 bool checkMergeStoreCandidatesForDependencies(
724 int64_t ElementSizeBytes)
const;
729 unsigned NumConsecutiveStores,
730 EVT MemVT,
SDNode *Root,
bool AllowVectors);
737 unsigned NumConsecutiveStores,
EVT MemVT,
743 unsigned NumConsecutiveStores,
EVT MemVT,
744 SDNode *Root,
bool AllowVectors,
745 bool IsNonTemporalStore,
bool IsNonTemporalLoad);
764 bool hasOperation(
unsigned Opcode,
EVT VT) {
776 EVT getShiftAmountTy(
EVT LHSTy) {
783 bool isTypeLegal(
const EVT &VT) {
784 if (!LegalTypes)
return true;
789 EVT getSetCCResultType(
EVT VT)
const {
804 explicit WorklistRemover(DAGCombiner &dc)
808 DC.removeFromWorklist(
N);
816 explicit WorklistInserter(DAGCombiner &dc)
821 void NodeInserted(
SDNode *
N)
override {
DC.ConsiderForPruning(
N); }
831 ((DAGCombiner*)
DC)->AddToWorklist(
N);
836 return ((DAGCombiner*)
DC)->CombineTo(
N, &To[0], To.
size(), AddTo);
841 return ((DAGCombiner*)
DC)->CombineTo(
N, Res, AddTo);
846 return ((DAGCombiner*)
DC)->CombineTo(
N, Res0, Res1, AddTo);
851 return ((DAGCombiner*)
DC)->recursivelyDeleteUnusedNodes(
N);
856 return ((DAGCombiner*)
DC)->CommitTargetLoweringOpt(TLO);
863 void DAGCombiner::deleteAndRecombine(
SDNode *
N) {
872 if (
Op->hasOneUse() ||
Op->getNumValues() > 1)
873 AddToWorklist(
Op.getNode());
893 SDValue &CC,
bool MatchStrict)
const {
895 LHS =
N.getOperand(0);
896 RHS =
N.getOperand(1);
897 CC =
N.getOperand(2);
904 LHS =
N.getOperand(1);
905 RHS =
N.getOperand(2);
906 CC =
N.getOperand(3);
918 LHS =
N.getOperand(0);
919 RHS =
N.getOperand(1);
920 CC =
N.getOperand(4);
927 bool DAGCombiner::isOneUseSetCC(
SDValue N)
const {
929 if (isSetCCEquivalent(
N, N0, N1, N2) &&
N.getNode()->hasOneUse())
944 MaskForTy = 0xFFFFULL;
947 MaskForTy = 0xFFFFFFFFULL;
966 return !(Const->isOpaque() && NoOpaques);
969 unsigned BitWidth =
N.getScalarValueSizeInBits();
974 if (!Const || Const->getAPIntValue().getBitWidth() !=
BitWidth ||
975 (Const->isOpaque() && NoOpaques))
994 !cast<ConstantSDNode>(
LD->getOperand(2))->isOpaque());
997 bool DAGCombiner::reassociationCanBreakAddressingModePattern(
unsigned Opc,
1013 auto *C1 = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
1014 auto *C2 = dyn_cast<ConstantSDNode>(N1);
1018 const APInt &C1APIntVal = C1->getAPIntValue();
1019 const APInt &C2APIntVal = C2->getAPIntValue();
1023 const APInt CombinedValueIntVal = C1APIntVal + C2APIntVal;
1026 const int64_t CombinedValue = CombinedValueIntVal.
getSExtValue();
1029 auto LoadStore = dyn_cast<MemSDNode>(Node);
1036 AM.
BaseOffs = C2APIntVal.getSExtValue();
1038 unsigned AS =
LoadStore->getAddressSpace();
1055 SDValue DAGCombiner::reassociateOpsCommutative(
unsigned Opc,
const SDLoc &
DL,
1069 return DAG.
getNode(Opc,
DL, VT, N00, OpNode);
1076 return DAG.
getNode(Opc,
DL, VT, OpNode, N01);
1094 if (
SDValue Combined = reassociateOpsCommutative(Opc,
DL, N0, N1))
1096 if (
SDValue Combined = reassociateOpsCommutative(Opc,
DL, N1, N0))
1103 assert(
N->getNumValues() == NumTo &&
"Broken CombineTo call!");
1107 dbgs() <<
" and " << NumTo - 1 <<
" other values\n");
1108 for (
unsigned i = 0,
e = NumTo; i !=
e; ++i)
1109 assert((!To[i].getNode() ||
1110 N->getValueType(i) == To[i].getValueType()) &&
1111 "Cannot combine value to value of different type!");
1113 WorklistRemover DeadNodes(*
this);
1117 for (
unsigned i = 0,
e = NumTo; i !=
e; ++i) {
1118 if (To[i].getNode()) {
1119 AddToWorklist(To[i].getNode());
1120 AddUsersToWorklist(To[i].getNode());
1129 deleteAndRecombine(
N);
1143 WorklistRemover DeadNodes(*
this);
1159 const APInt &DemandedElts,
1160 bool AssumeSingleUse) {
1168 AddToWorklist(
Op.getNode());
1170 CommitTargetLoweringOpt(TLO);
1177 bool DAGCombiner::SimplifyDemandedVectorElts(
SDValue Op,
1178 const APInt &DemandedElts,
1179 bool AssumeSingleUse) {
1181 APInt KnownUndef, KnownZero;
1183 TLO, 0, AssumeSingleUse))
1187 AddToWorklist(
Op.getNode());
1189 CommitTargetLoweringOpt(TLO);
1193 void DAGCombiner::ReplaceLoadWithPromotedLoad(
SDNode *
Load,
SDNode *ExtLoad) {
1195 EVT VT =
Load->getValueType(0);
1200 WorklistRemover DeadNodes(*
this);
1203 deleteAndRecombine(
Load);
1204 AddToWorklist(Trunc.
getNode());
1212 EVT MemVT =
LD->getMemoryVT();
1214 :
LD->getExtensionType();
1217 LD->getChain(),
LD->getBasePtr(),
1218 MemVT,
LD->getMemOperand());
1221 unsigned Opc =
Op.getOpcode();
1225 if (
SDValue Op0 = SExtPromoteOperand(
Op.getOperand(0), PVT))
1229 if (
SDValue Op0 = ZExtPromoteOperand(
Op.getOperand(0), PVT))
1247 EVT OldVT =
Op.getValueType();
1249 bool Replace =
false;
1250 SDValue NewOp = PromoteOperand(
Op, PVT, Replace);
1253 AddToWorklist(NewOp.
getNode());
1256 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1262 EVT OldVT =
Op.getValueType();
1264 bool Replace =
false;
1265 SDValue NewOp = PromoteOperand(
Op, PVT, Replace);
1268 AddToWorklist(NewOp.
getNode());
1271 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1279 if (!LegalOperations)
1282 EVT VT =
Op.getValueType();
1288 unsigned Opc =
Op.getOpcode();
1296 assert(PVT != VT &&
"Don't know what type to promote to!");
1300 bool Replace0 =
false;
1302 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1304 bool Replace1 =
false;
1306 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1318 Replace1 &= (N0 != N1) && !N1->
hasOneUse();
1321 CombineTo(
Op.getNode(), RV);
1347 if (!LegalOperations)
1350 EVT VT =
Op.getValueType();
1356 unsigned Opc =
Op.getOpcode();
1364 assert(PVT != VT &&
"Don't know what type to promote to!");
1368 bool Replace =
false;
1372 N0 = SExtPromoteOperand(N0, PVT);
1374 N0 = ZExtPromoteOperand(N0, PVT);
1376 N0 = PromoteOperand(N0, PVT, Replace);
1386 ReplaceLoadWithPromotedLoad(
Op.getOperand(0).getNode(), N0.
getNode());
1396 if (!LegalOperations)
1399 EVT VT =
Op.getValueType();
1405 unsigned Opc =
Op.getOpcode();
1413 assert(PVT != VT &&
"Don't know what type to promote to!");
1423 bool DAGCombiner::PromoteLoad(
SDValue Op) {
1424 if (!LegalOperations)
1430 EVT VT =
Op.getValueType();
1436 unsigned Opc =
Op.getOpcode();
1444 assert(PVT != VT &&
"Don't know what type to promote to!");
1449 EVT MemVT =
LD->getMemoryVT();
1451 :
LD->getExtensionType();
1453 LD->getChain(),
LD->getBasePtr(),
1454 MemVT,
LD->getMemOperand());
1458 Result.getNode()->dump(&DAG);
dbgs() <<
'\n');
1459 WorklistRemover DeadNodes(*
this);
1462 deleteAndRecombine(
N);
1463 AddToWorklist(
Result.getNode());
1475 bool DAGCombiner::recursivelyDeleteUnusedNodes(
SDNode *
N) {
1476 if (!
N->use_empty())
1486 if (
N->use_empty()) {
1487 for (
const SDValue &ChildN :
N->op_values())
1488 Nodes.
insert(ChildN.getNode());
1495 }
while (!Nodes.
empty());
1510 WorklistInserter AddNodes(*
this);
1514 AddToWorklist(&Node);
1522 while (
SDNode *
N = getNextWorklistEntry()) {
1526 if (recursivelyDeleteUnusedNodes(
N))
1529 WorklistRemover DeadNodes(*
this);
1537 for (
SDNode *LN : UpdatedNodes)
1538 AddToWorklistWithUsers(LN);
1550 for (
const SDValue &ChildN :
N->op_values())
1551 if (!CombinedNodes.
count(ChildN.getNode()))
1552 AddToWorklist(ChildN.getNode());
1570 "Node was deleted but visit returned new node!");
1578 N->getNumValues() == 1 &&
"Type mismatch");
1589 AddUsersToWorklist(RV.
getNode());
1596 recursivelyDeleteUnusedNodes(
N);
1605 switch (
N->getOpcode()) {
1741 #define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) case ISD::SDOPC:
1742 #include "llvm/IR/VPIntrinsics.def"
1743 return visitVPOp(
N);
1750 if (!DisableGenericCombines)
1756 "Node was deleted but visit returned NULL!");
1763 DagCombineInfo(DAG,
Level,
false,
this);
1771 switch (
N->getOpcode()) {
1779 RV = PromoteIntBinOp(
SDValue(
N, 0));
1784 RV = PromoteIntShiftOp(
SDValue(
N, 0));
1801 N->getNumValues() == 1) {
1806 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
1821 if (
unsigned NumOps =
N->getNumOperands()) {
1822 if (
N->getOperand(0).getValueType() ==
MVT::Other)
1823 return N->getOperand(0);
1824 if (
N->getOperand(NumOps-1).getValueType() ==
MVT::Other)
1825 return N->getOperand(NumOps-1);
1826 for (
unsigned i = 1; i < NumOps-1; ++i)
1827 if (
N->getOperand(i).getValueType() ==
MVT::Other)
1828 return N->getOperand(i);
1836 if (
N->getNumOperands() == 2) {
1838 return N->getOperand(0);
1840 return N->getOperand(1);
1855 AddToWorklist(*(
N->use_begin()));
1860 bool Changed =
false;
1867 for (
unsigned i = 0; i < TFs.
size(); ++i) {
1872 for (
unsigned j = i; j < TFs.
size(); j++)
1883 switch (
Op.getOpcode()) {
1901 if (SeenOps.
insert(
Op.getNode()).second)
1912 for (
unsigned i = 1,
e = TFs.
size(); i <
e; i++)
1913 AddToWorklist(TFs[i]);
1925 bool DidPruneOps =
false;
1927 unsigned NumLeftToConsider = 0;
1929 Worklist.
push_back(std::make_pair(
Op.getNode(), NumLeftToConsider++));
1933 auto AddToWorklist = [&](
unsigned CurIdx,
SDNode *
Op,
unsigned OpNumber) {
1939 unsigned OrigOpNumber = 0;
1940 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() !=
Op)
1942 assert((OrigOpNumber != Ops.size()) &&
1943 "expected to find TokenFactor Operand");
1945 for (
unsigned i = CurIdx + 1; i < Worklist.
size(); ++i) {
1946 if (Worklist[i].second == OrigOpNumber) {
1947 Worklist[i].second = OpNumber;
1950 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
1951 OpWorkCount[OrigOpNumber] = 0;
1952 NumLeftToConsider--;
1955 if (SeenChains.
insert(
Op).second) {
1956 OpWorkCount[OpNumber]++;
1961 for (
unsigned i = 0; i < Worklist.
size() && i < 1024; ++i) {
1963 if (NumLeftToConsider <= 1)
1965 auto CurNode = Worklist[i].first;
1966 auto CurOpNumber = Worklist[i].second;
1967 assert((OpWorkCount[CurOpNumber] > 0) &&
1968 "Node should not appear in worklist");
1969 switch (CurNode->getOpcode()) {
1975 NumLeftToConsider++;
1978 for (
const SDValue &
Op : CurNode->op_values())
1979 AddToWorklist(i,
Op.getNode(), CurOpNumber);
1985 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
1988 if (
auto *MemNode = dyn_cast<MemSDNode>(CurNode))
1989 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
1992 OpWorkCount[CurOpNumber]--;
1993 if (OpWorkCount[CurOpNumber] == 0)
1994 NumLeftToConsider--;
2008 if (SeenChains.
count(
Op.getNode()) == 0)
2023 WorklistRemover DeadNodes(*
this);
2029 AddUsersToWorklist(
N);
2033 for (
unsigned i = 0,
e =
N->getNumOperands(); i !=
e; ++i)
2036 }
while (!
N->use_empty());
2037 deleteAndRecombine(
N);
2045 return Const !=
nullptr && !Const->isOpaque() ? Const :
nullptr;
2056 if (
LD->isIndexed() ||
LD->getBasePtr().getNode() !=
N)
2058 VT =
LD->getMemoryVT();
2059 AS =
LD->getAddressSpace();
2061 if (
ST->isIndexed() ||
ST->getBasePtr().getNode() !=
N)
2063 VT =
ST->getMemoryVT();
2064 AS =
ST->getAddressSpace();
2066 if (
LD->isIndexed() ||
LD->getBasePtr().getNode() !=
N)
2068 VT =
LD->getMemoryVT();
2069 AS =
LD->getAddressSpace();
2071 if (
ST->isIndexed() ||
ST->getBasePtr().getNode() !=
N)
2073 VT =
ST->getMemoryVT();
2074 AS =
ST->getAddressSpace();
2084 AM.BaseOffs =
Offset->getSExtValue();
2088 }
else if (
N->getOpcode() ==
ISD::SUB) {
2089 AM.HasBaseReg =
true;
2093 AM.BaseOffs = -
Offset->getSExtValue();
2109 bool ShouldCommuteOperands) {
2114 if (ShouldCommuteOperands)
2121 unsigned Opcode =
N->getOpcode();
2122 EVT VT =
N->getValueType(0);
2130 auto isIdentityConstantForOpcode = [](
unsigned Opcode,
SDValue V) {
2134 return C->isZero() &&
C->isNegative();
2136 return C->isZero() && !
C->isNegative();
2139 return C->isExactlyValue(1.0);
2147 if (isIdentityConstantForOpcode(Opcode, TVal)) {
2153 if (isIdentityConstantForOpcode(Opcode, FVal)) {
2164 "Unexpected binary operator");
2181 unsigned SelOpNo = 0;
2206 bool CanFoldNonConst =
2212 if (!CanFoldNonConst &&
2222 : DAG.
getNode(BinOpcode,
DL, VT, CT, CBO);
2223 if (!CanFoldNonConst && !NewCT.
isUndef() &&
2229 : DAG.
getNode(BinOpcode,
DL, VT, CF, CBO);
2230 if (!CanFoldNonConst && !NewCF.
isUndef() &&
2242 "Expecting add or sub");
2247 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2248 SDValue C = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2249 SDValue Z = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2250 auto *CN = dyn_cast<ConstantSDNode>(
C);
2255 if (Z.getOperand(0).getOpcode() !=
ISD::SETCC ||
2256 Z.getOperand(0).getValueType() !=
MVT::i1)
2260 SDValue SetCC = Z.getOperand(0);
2271 EVT VT =
C.getValueType();
2283 "Expecting add or sub");
2287 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2288 SDValue ConstantOp = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2289 SDValue ShiftOp = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2314 {ConstantOp, DAG.getConstant(1, DL, VT)}))
2345 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
2363 assert(Sub &&
"Constant folding failed");
2372 assert(Add &&
"Constant folding failed");
2383 if ((!LegalOperations ||
2386 X.getScalarValueSizeInBits() == 1) {
2398 {N1, N0.getOperand(1)}))
2403 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
2407 if (!reassociationCanBreakAddressingModePattern(
ISD::ADD,
DL, N0, N1)) {
2423 if (
SDValue Add = ReassociateAddOr(N0, N1))
2425 if (
SDValue Add = ReassociateAddOr(N1, N0))
2491 return (!Max && !
Op) ||
2492 (
Max &&
Op &&
Max->getAPIntValue() == (-
Op->getAPIntValue()));
2544 if (
SDValue Combined = visitADDLikeCommutative(N0, N1,
N))
2547 if (
SDValue Combined = visitADDLikeCommutative(N1, N0,
N))
2559 if (
SDValue Combined = visitADDLike(
N))
2595 APInt NewStep = C0 + C1;
2605 APInt NewStep = SV0 + SV1;
2614 unsigned Opcode =
N->getOpcode();
2631 return DAG.
getNode(Opcode,
DL, VT, N1, N0);
2655 bool Masked =
false;
2787 DAG.
getVTList(VT, Carry.getValueType()), N0,
2800 if (!
N->hasAnyUseOfValue(1))
2833 if (Force && isa<ConstantSDNode>(V))
2845 bool IsFlip =
false;
2848 IsFlip = Const->isOne();
2851 IsFlip = Const->isAllOnes();
2854 IsFlip = (Const->getAPIntValue() & 0x01) == 1;
2871 EVT CarryVT =
N->getValueType(1);
2875 if (!
N->hasAnyUseOfValue(1))
2882 return DAG.
getNode(
N->getOpcode(),
DL,
N->getVTList(), N1, N0);
2902 if (
SDValue Combined = visitUADDOLike(N0, N1,
N))
2905 if (
SDValue Combined = visitUADDOLike(N1, N0,
N))
2939 SDValue CarryIn =
N->getOperand(2);
2958 SDValue CarryIn =
N->getOperand(2);
2969 if (!LegalOperations ||
2979 AddToWorklist(CarryExt.
getNode());
2985 if (
SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn,
N))
2988 if (
SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn,
N))
2997 SDValue CarryIn =
N->getOperand(2);
3008 if (!LegalOperations ||
3150 unsigned CarryInOperandNum =
3152 if (Opcode ==
ISD::USUBO && CarryInOperandNum != 1)
3237 "Illegal truncation");
3263 !(!LegalOperations || hasOperation(
ISD::USUBSAT, DstVT)))
3266 EVT SubVT =
N->getValueType(0);
3335 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
3343 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
3370 if (
N->getFlags().hasNoUnsignedWrap())
3376 if (
N->getFlags().hasNoSignedWrap())
3427 assert(NewC &&
"Constant folding failed");
3437 assert(NewC &&
"Constant folding failed");
3448 assert(NewC &&
"Constant folding failed");
3458 assert(NewC &&
"Constant folding failed");
3535 if (
SDValue V = foldSubToUSubSat(VT,
N))
3597 if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0))
3614 if (GA->getGlobal() == GB->getGlobal())
3661 DAG.
getVTList(VT, Carry.getValueType()), NegX, Zero,
3710 if (!
N->hasAnyUseOfValue(1))
3737 EVT CarryVT =
N->getValueType(1);
3741 if (!
N->hasAnyUseOfValue(1))
3773 SDValue CarryIn =
N->getOperand(2);
3785 SDValue CarryIn =
N->getOperand(2);
3789 if (!LegalOperations ||
3800 SDValue CarryIn =
N->getOperand(2);
3804 if (!LegalOperations ||
3854 bool N1IsConst =
false;
3855 bool N1IsOpaqueConst =
false;
3866 "Splat APInt should be element width");
3868 N1IsConst = isa<ConstantSDNode>(N1);
3870 ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
3871 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
3876 if (N1IsConst && ConstValue1.
isZero())
3880 if (N1IsConst && ConstValue1.
isOne())
3883 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
3887 if (N1IsConst && ConstValue1.
isAllOnes()) {
3898 SDValue LogBase2 = BuildLogBase2(N1,
DL);
3906 unsigned Log2Val = (-ConstValue1).logBase2();
3941 if ((MulC - 1).isPowerOf2())
3943 else if ((MulC + 1).isPowerOf2())
3948 MathOp ==
ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2();
3951 "multiply-by-constant generated out of bounds shift");
3956 TZeros ? DAG.
getNode(MathOp,
DL, VT, Shl,
4012 const APInt &C1 = NC1->getAPIntValue();
4021 APInt NewStep = C0 * MulVal;
4034 if (!V || V->isZero()) {
4049 for (
unsigned I = 0;
I != NumElts; ++
I)
4070 switch (
NodeType.getSimpleVT().SimpleTy) {
4071 default:
return false;
4072 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
4073 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
4074 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
4075 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
4076 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
4084 if (Node->use_empty())
4087 unsigned Opcode = Node->getOpcode();
4092 EVT VT = Node->getValueType(0);
4106 unsigned OtherOpcode = 0;
4117 SDValue Op0 = Node->getOperand(0);
4118 SDValue Op1 = Node->getOperand(1);
4127 unsigned UserOpc =
User->getOpcode();
4128 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
4132 if (UserOpc == OtherOpcode) {
4134 combined = DAG.
getNode(DivRemOpc,
SDLoc(Node), VTs, Op0, Op1);
4135 }
else if (UserOpc == DivRemOpc) {
4138 assert(UserOpc == Opcode);
4143 CombineTo(
User, combined);
4154 EVT VT =
N->getValueType(0);
4157 unsigned Opc =
N->getOpcode();
4166 if (DAG.
isUndef(Opc, {N0, N1}))
4177 if (N0C && N0C->
isZero())
4200 EVT VT =
N->getValueType(0);
4210 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4227 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4235 if (
SDValue V = visitSDIVLike(N0, N1,
N)) {
4242 AddToWorklist(
Mul.getNode());
4244 CombineTo(RemNode, Sub);
4262 EVT VT =
N->getValueType(0);
4269 if (
C->isZero() ||
C->isOpaque())
4271 if (
C->getAPIntValue().isPowerOf2())
4273 if (
C->getAPIntValue().isNegatedPowerOf2())
4299 AddToWorklist(Sign.
getNode());
4305 AddToWorklist(
Add.getNode());
4316 Sra = DAG.
getSelect(
DL, VT, IsOneOrAllOnes, N0, Sra);
4344 EVT VT =
N->getValueType(0);
4354 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4367 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4370 if (
SDValue V = visitUDIVLike(N0, N1,
N)) {
4377 AddToWorklist(
Mul.getNode());
4379 CombineTo(RemNode, Sub);
4397 EVT VT =
N->getValueType(0);
4402 SDValue LogBase2 = BuildLogBase2(N1,
DL);
4403 AddToWorklist(LogBase2.
getNode());
4407 AddToWorklist(Trunc.
getNode());
4416 SDValue LogBase2 = BuildLogBase2(N10,
DL);
4417 AddToWorklist(LogBase2.
getNode());
4421 AddToWorklist(Trunc.
getNode());
4423 AddToWorklist(
Add.getNode());
4440 unsigned Opcode =
N->getOpcode();
4443 EVT VT =
N->getValueType(0);
4455 if (!isSigned && N1C && N1C->
isAllOnes())
4462 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4475 AddToWorklist(
Add.getNode());
4483 AddToWorklist(
Add.getNode());
4499 isSigned ? visitSDIVLike(N0, N1,
N) : visitUDIVLike(N0, N1,
N);
4505 CombineTo(DivNode, OptimizedDiv);
4508 AddToWorklist(OptimizedDiv.
getNode());
4509 AddToWorklist(
Mul.getNode());
4516 return DivRem.getValue(1);
4524 EVT VT =
N->getValueType(0);
4537 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4584 EVT VT =
N->getValueType(0);
4597 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4622 SDValue LogBase2 = BuildLogBase2(N1,
DL);
4660 SDValue DAGCombiner::SimplifyNodeWithTwoResults(
SDNode *
N,
unsigned LoOp,
4663 bool HiExists =
N->hasAnyUseOfValue(1);
4664 if (!HiExists && (!LegalOperations ||
4667 return CombineTo(
N, Res, Res);
4671 bool LoExists =
N->hasAnyUseOfValue(0);
4672 if (!LoExists && (!LegalOperations ||
4675 return CombineTo(
N, Res, Res);
4679 if (LoExists && HiExists)
4685 AddToWorklist(
Lo.getNode());
4688 (!LegalOperations ||
4690 return CombineTo(
N, LoOpt, LoOpt);
4695 AddToWorklist(
Hi.getNode());
4698 (!LegalOperations ||
4700 return CombineTo(
N, HiOpt, HiOpt);
4710 EVT VT =
N->getValueType(0);
4730 return CombineTo(
N,
Lo,
Hi);
4741 EVT VT =
N->getValueType(0);
4747 return CombineTo(
N, Zero, Zero);
4753 return CombineTo(
N,
N->getOperand(0), Zero);
4773 return CombineTo(
N,
Lo,
Hi);
4786 EVT CarryVT =
N->getValueType(1);
4807 return DAG.
getNode(
N->getOpcode(),
DL,
N->getVTList(), N1, N0);
4817 N->getVTList(), N0, N0);
4823 return CombineTo(
N, And,
4879 unsigned Opcode0 = isSignedMinMax(N0, N1, N2, N3, CC);
4897 N0CC = cast<CondCodeSDNode>(N0.
getOperand(4))->get();
4913 unsigned Opcode1 = isSignedMinMax(N00, N01, N02, N03, N0CC);
4914 if (!Opcode1 || Opcode0 == Opcode1)
4924 APInt MinCPlus1 = MinC + 1;
4925 if (-MaxC == MinCPlus1 && MinCPlus1.
isPowerOf2()) {
4983 unsigned BW = (C1 + 1).exactLogBase2();
5003 unsigned Opcode =
N->getOpcode();
5013 return DAG.
getNode(Opcode,
DL, VT, N1, N0);
5017 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
5034 return DAG.
getNode(AltOpcode,
DL, VT, N0, N1);
5054 SDValue DAGCombiner::hoistLogicOpWithSameOpcodeHands(
SDNode *
N) {
5055 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
5057 unsigned LogicOpcode =
N->getOpcode();
5060 LogicOpcode ==
ISD::XOR) &&
"Expected logic opcode");
5073 EVT XVT =
X.getValueType();
5082 if (XVT !=
Y.getValueType())
5086 if ((VT.
isVector() || LegalOperations) &&
5096 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
5106 if (XVT !=
Y.getValueType())
5118 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
5139 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
5152 if (XVT.
isInteger() && XVT ==
Y.getValueType() &&
5156 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
5173 auto *SVN0 = cast<ShuffleVectorSDNode>(N0);
5174 auto *SVN1 = cast<ShuffleVectorSDNode>(N1);
5175 assert(
X.getValueType() ==
Y.getValueType() &&
5176 "Inputs to shuffles are not the same type");
5182 if (!SVN0->hasOneUse() || !SVN1->hasOneUse() ||
5183 !SVN0->getMask().equals(SVN1->getMask()))
5219 SDValue LL, LR, RL, RR, N0CC, N1CC;
5220 if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
5221 !isSetCCEquivalent(N1, RL, RR, N1CC))
5225 "Unexpected operand types for bitwise logic op");
5228 "Unexpected operand types for setcc");
5244 if (LR == RR && CC0 == CC1 && IsInteger) {
5249 bool AndEqZero = IsAnd && CC1 ==
ISD::SETEQ && IsZero;
5251 bool AndGtNeg1 = IsAnd && CC1 ==
ISD::SETGT && IsNeg1;
5253 bool OrNeZero = !IsAnd && CC1 ==
ISD::SETNE && IsZero;
5255 bool OrLtZero = !IsAnd && CC1 ==
ISD::SETLT && IsZero;
5261 if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
5263 AddToWorklist(
Or.getNode());
5268 bool AndEqNeg1 = IsAnd && CC1 ==
ISD::SETEQ && IsNeg1;
5270 bool AndLtZero = IsAnd && CC1 ==
ISD::SETLT && IsZero;
5272 bool OrNeNeg1 = !IsAnd && CC1 ==
ISD::SETNE && IsNeg1;
5274 bool OrGtNeg1 = !IsAnd && CC1 ==
ISD::SETGT && IsNeg1;
5280 if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
5282 AddToWorklist(
And.getNode());
5296 AddToWorklist(
Add.getNode());
5326 if ((CMax - CMin).isPowerOf2()) {
5343 if (LL == RR && LR == RL) {
5350 if (LL == RL && LR == RR) {
5354 (!LegalOperations ||
5375 if (
SDValue V = foldLogicOfSetCCs(
true, N0, N1,
DL))
5388 APInt SRLC = SRLI->getAPIntValue();
5389 if (
ADDC.getMinSignedBits() <= 64 &&
5401 CombineTo(N0.
getNode(), NewAdd);
5418 const APInt &AndMask = CAnd->getAPIntValue();
5431 (ShiftBits + MaskBits <=
Size / 2) &&
5464 EVT LoadResultTy,
EVT &ExtVT) {
5473 if (ExtVT == LoadedVT &&
5474 (!LegalOperations ||
5490 if (LegalOperations &&
5500 bool DAGCombiner::isLegalNarrowLdSt(
LSBaseSDNode *LDST,
5526 if (LdStMemVT.
bitsLT(MemVT))
5531 assert(ShAmt % 8 == 0 &&
"ShAmt is byte offset");
5532 const unsigned ByteShAmt = ShAmt / 8;
5546 if (isa<LoadSDNode>(LDST)) {
5553 if (LegalOperations &&
5562 if (
Load->getNumValues() > 2)
5575 assert(isa<StoreSDNode>(LDST) &&
"It is not a Load nor a Store SDNode");
5581 if (LegalOperations &&
5588 bool DAGCombiner::SearchForAndLoads(
SDNode *
N,
5596 if (
Op.getValueType().isVector())
5600 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
5602 (
Mask->getAPIntValue() &
C->getAPIntValue()) !=
C->getAPIntValue())
5607 if (!
Op.hasOneUse())
5610 switch(
Op.getOpcode()) {
5612 auto *
Load = cast<LoadSDNode>(
Op);
5614 if (isAndLoadExtLoad(
Mask,
Load,
Load->getValueType(0), ExtVT) &&
5632 unsigned ActiveBits =
Mask->getAPIntValue().countTrailingOnes();
5635 cast<VTSDNode>(
Op.getOperand(1))->getVT() :
5636 Op.getOperand(0).getValueType();
5647 if (!SearchForAndLoads(
Op.getNode(), Loads, NodesWithConsts,
Mask,
5658 NodeToMask =
Op.getNode();
5661 for (
unsigned i = 0,
e = NodeToMask->
getNumValues(); i <
e; ++i) {
5665 NodeToMask =
nullptr;
5677 bool DAGCombiner::BackwardsPropagateMask(
SDNode *
N) {
5678 auto *
Mask = dyn_cast<ConstantSDNode>(
N->getOperand(1));
5682 if (!
Mask->getAPIntValue().isMask())
5686 if (isa<LoadSDNode>(
N->getOperand(0)))
5691 SDNode *FixupNode =
nullptr;
5692 if (SearchForAndLoads(
N, Loads, NodesWithConsts,
Mask, FixupNode)) {
5693 if (Loads.
size() == 0)
5705 SDValue(FixupNode, 0), MaskOp);
5712 for (
auto *LogicN : NodesWithConsts) {
5716 if (isa<ConstantSDNode>(Op0))
5726 for (
auto *
Load : Loads) {
5734 SDValue NewLoad = reduceLoadWidth(
And.getNode());
5736 "Shouldn't be masking the load if it can't be narrowed");
5750 SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(
SDNode *
N) {
5761 unsigned OuterShift;
5762 unsigned InnerShift;
5764 auto matchMask = [&OuterShift, &InnerShift, &
Y](
SDValue M) ->
bool {
5767 OuterShift =
M->getOpcode();
5776 Y =
M->getOperand(1);
5783 else if (matchMask(N0))
5789 EVT VT =
N->getValueType(0);
5806 EVT VT =
And->getValueType(0);
5813 SDValue Not =
And->getOperand(0), And1 =
And->getOperand(1);
5836 if (ShiftAmt.
uge(VTBitWidth))
5919 auto *MLoad = dyn_cast<MaskedLoadSDNode>(N0);
5920 auto *BVec = dyn_cast<BuildVectorSDNode>(N1);
5921 if (MLoad && BVec && MLoad->getExtensionType() ==
ISD::EXTLOAD &&
5923 EVT LoadVT = MLoad->getMemoryVT();
5932 if (Splat->getAPIntValue().isMask(ElementSize)) {
5934 ExtVT,
SDLoc(
N), MLoad->getChain(), MLoad->getBasePtr(),
5935 MLoad->getOffset(), MLoad->getMask(), MLoad->getPassThru(),
5936 LoadVT, MLoad->getMemOperand(), MLoad->getAddressingMode(),
5954 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
5963 if (
SDValue Shuffle = XformToShuffleWithZero(
N))
5971 return RHS->getAPIntValue().isSubsetOf(
LHS->getAPIntValue());
5991 CombineTo(N0.
getNode(), Zext);
6017 APInt SplatValue, SplatUndef;
6018 unsigned SplatBitSize;
6020 bool IsSplat =
Vector->isConstantSplat(SplatValue, SplatUndef,
6021 SplatBitSize, HasAnyUndefs);
6025 SplatValue |= SplatUndef;
6031 unsigned EltBitWidth =
Vector->getValueType(0).getScalarSizeInBits();
6036 if (EltBitWidth > SplatBitSize)
6037 for (SplatValue = SplatValue.
zextOrTrunc(EltBitWidth);
6038 SplatBitSize < EltBitWidth; SplatBitSize = SplatBitSize * 2)
6039 SplatValue |= SplatValue.
shl(SplatBitSize);
6043 if ((SplatBitSize % EltBitWidth) == 0) {
6045 for (
unsigned i = 0, n = (SplatBitSize / EltBitWidth); i < n; ++i)
6055 Load->getValueType(0),
6056 Load->getMemoryVT());
6064 switch (
Load->getExtensionType()) {
6065 default:
B =
false;
break;
6082 Load->getChain(),
Load->getBasePtr(),
6083 Load->getOffset(),
Load->getMemoryVT(),
6084 Load->getMemOperand());
6086 if (
Load->getNumValues() == 3) {
6090 CombineTo(
Load, To, 3,
true);
6092 CombineTo(
Load, NewLoad.getValue(0), NewLoad.getValue(1));
6101 if (
auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
6102 EVT MemVT = GN0->getMemoryVT();
6108 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
6109 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
6115 CombineTo(
N, ZExtLoad);
6116 AddToWorklist(ZExtLoad.
getNode());
6128 if (
SDValue Res = reduceLoadWidth(
N)) {
6142 if (BackwardsPropagateMask(
N))
6146 if (
SDValue Combined = visitANDLike(N0, N1,
N))
6151 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
6191 ((!LegalOperations && LN0->
isSimple()) ||
6209 if (
SDValue Shifts = unfoldExtremeBitClearingToShifts(
N))
6226 auto *
C = dyn_cast<ConstantSDNode>(
RHS);
6230 if (!
C->getAPIntValue().isMask(
6231 LHS.getOperand(0).getValueType().getFixedSizeInBits()))
6238 if (IsAndZeroExtMask(N0, N1))
6250 bool DemandHighBits) {
6251 if (!LegalOperations)
6254 EVT VT =
N->getValueType(0);
6261 bool LookPassAnd0 =
false;
6262 bool LookPassAnd1 =
false;
6277 LookPassAnd0 =
true;
6287 LookPassAnd1 =
true;
6313 LookPassAnd0 =
true;
6327 LookPassAnd1 =
true;
6336 if (DemandHighBits && OpSizeInBits > 16) {
6345 if (!LookPassAnd1 &&
6352 if (OpSizeInBits > 16) {
6368 if (!
N.getNode()->hasOneUse())
6371 unsigned Opc =
N.getOpcode();
6383 N1C = dyn_cast<ConstantSDNode>(
N.getOperand(1));
6385 N1C = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
6389 unsigned MaskByteOffset;
6393 case 0xFF: MaskByteOffset = 0;
break;
6394 case 0xFF00: MaskByteOffset = 1;
break;
6403 case 0xFF0000: MaskByteOffset = 2;
break;
6404 case 0xFF000000: MaskByteOffset = 3;
break;
6409 if (MaskByteOffset == 0 || MaskByteOffset == 2) {
6415 if (!
C ||
C->getZExtValue() != 8)
6423 if (!
C ||
C->getZExtValue() != 8)
6429 if (MaskByteOffset != 0 && MaskByteOffset != 2)
6432 if (!
C ||
C->getZExtValue() != 8)
6437 if (MaskByteOffset != 1 && MaskByteOffset != 3)
6440 if (!
C ||
C->getZExtValue() != 8)
6444 if (Parts[MaskByteOffset])
6459 if (!
C ||
C->getAPIntValue() != 16)
6461 Parts[0] = Parts[1] =
N.getOperand(0).getOperand(0).getNode();
6476 "MatchBSwapHWordOrAndAnd: expecting i32");
6486 if (!Mask0 || !Mask1)
6497 if (!ShiftAmt0 || !ShiftAmt1)
6517 if (!LegalOperations)
6520 EVT VT =
N->getValueType(0);
6559 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
6588 if (
SDValue V = foldLogicOfSetCCs(
false, N0, N1,
DL))
6603 const APInt &LHSMask = N0O1C->getAPIntValue();
6604 const APInt &RHSMask = N1O1C->getAPIntValue();
6682 if (isa<ShuffleVectorSDNode>(N0) &&
6683 isa<ShuffleVectorSDNode>(N1) &&
6691 if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
6692 assert((!ZeroN00 || !ZeroN01) &&
"Both inputs zero!");
6693 assert((!ZeroN10 || !ZeroN11) &&
"Both inputs zero!");
6696 bool CanFold =
true;
6700 for (
int i = 0; i != NumElts; ++i) {
6705 bool M0Zero =
M0 < 0 || (ZeroN00 == (
M0 < NumElts));
6706 bool M1Zero =
M1 < 0 || (ZeroN10 == (
M1 < NumElts));
6710 if ((M0Zero &&
M1 < 0) || (M1Zero &&
M0 < 0)) {
6716 if (M0Zero == M1Zero) {
6721 assert((
M0 >= 0 ||
M1 >= 0) &&
"Undef index!");
6727 Mask[i] = M1Zero ?
M0 % NumElts : (
M1 % NumElts) + NumElts;
6738 return LegalShuffle;
6752 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
6760 if (
SDValue Combined = visitORLike(N0, N1,
N))
6767 if (
SDValue BSwap = MatchBSwapHWord(
N, N0, N1))
6769 if (
SDValue BSwap = MatchBSwapHWordLow(
N, N0, N1))
6798 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
6815 if (
SDValue Combined = visitADDLike(
N))
6825 return Op.getOperand(0);
6868 assert(OppShift && ExtractFrom &&
"Empty SDValue");
6871 "Existing shift must be valid as a rotate half");
6897 bool IsMulOrDiv =
false;
6900 auto SelectOpcode = [&](
unsigned NeededShift,
unsigned MulOrDivVariant) {
6901 IsMulOrDiv = ExtractFrom.
getOpcode() == MulOrDivVariant;
6902 if (!IsMulOrDiv && ExtractFrom.
getOpcode() != NeededShift)
6904 Opcode = NeededShift;
6954 if (Rem != 0 || ResultAmt != OppLHSAmt)
6960 if (OppLHSAmt != ExtractFromAmt - NeededShiftAmt.
zextOrTrunc(
6969 return DAG.
getNode(Opcode,
DL, ResVT, OppShiftLHS, NewShiftNode);
7020 unsigned MaskLoBits = 0;
7025 if (NegC->getAPIntValue().getActiveBits() <=
Bits &&
7026 ((NegC->getAPIntValue() | Known.
Zero).countTrailingOnes() >=
Bits)) {
7046 if (PosC->getAPIntValue().getActiveBits() <= MaskLoBits &&
7047 ((PosC->getAPIntValue() | Known.
Zero).countTrailingOnes() >=
7066 if ((Pos == NegOp1) ||
7090 return Width.getLoBits(MaskLoBits) == 0;
7091 return Width == EltSize;
7101 SDValue InnerNeg,
unsigned PosOpcode,
7102 unsigned NegOpcode,
const SDLoc &
DL) {
7114 return DAG.
getNode(HasPos ? PosOpcode : NegOpcode,
DL, VT, Shifted,
7115 HasPos ? Pos : Neg);
7129 SDValue InnerNeg,
unsigned PosOpcode,
7130 unsigned NegOpcode,
const SDLoc &
DL) {
7141 if (
matchRotateSub(InnerPos, InnerNeg, EltBits, DAG, N0 == N1)) {
7143 return DAG.
getNode(HasPos ? PosOpcode : NegOpcode,
DL, VT, N0, N1,
7144 HasPos ? Pos : Neg);
7151 auto IsBinOpImm = [](
SDValue Op,
unsigned BinOpc,
unsigned Imm) {
7152 if (
Op.getOpcode() != BinOpc)
7161 IsBinOpImm(InnerNeg,
ISD::XOR, EltBits - 1) &&
7170 IsBinOpImm(InnerPos,
ISD::XOR, EltBits - 1) &&
7180 IsBinOpImm(InnerPos,
ISD::XOR, EltBits - 1) &&
7195 EVT VT =
LHS.getValueType();
7200 bool HasROTL = hasOperation(
ISD::ROTL, VT);
7201 bool HasROTR = hasOperation(
ISD::ROTR, VT);
7202 bool HasFSHL = hasOperation(
ISD::FSHL, VT);
7203 bool HasFSHR = hasOperation(
ISD::FSHR, VT);
7204 if (LegalOperations && !HasROTL && !HasROTR && !HasFSHL && !HasFSHR)
7209 LHS.getOperand(0).getValueType() ==
RHS.getOperand(0).getValueType()) {
7211 if (
SDValue Rot = MatchRotate(
LHS.getOperand(0),
RHS.getOperand(0),
DL)) {
7226 if (!LHSShift && !RHSShift)
7241 RHSShift = NewRHSShift;
7246 LHSShift = NewLHSShift;
7249 if (!RHSShift || !LHSShift)
7259 if (!IsRotate && !(HasFSHL || HasFSHR))
7282 return (
LHS->getAPIntValue() +
RHS->getAPIntValue()) == EltSizeInBits;
7286 if (IsRotate && (HasROTL || HasROTR || !(HasFSHL || HasFSHR))) {
7287 bool UseROTL = !LegalOperations || HasROTL;
7289 UseROTL ? LHSShiftAmt : RHSShiftAmt);
7291 bool UseFSHL = !LegalOperations || HasFSHL;
7293 RHSShiftArg, UseFSHL ? LHSShiftAmt : RHSShiftAmt);
7320 if (!HasROTL && !HasROTR && !HasFSHL && !HasFSHR)
7329 SDValue LExtOp0 = LHSShiftAmt;
7330 SDValue RExtOp0 = RHSShiftAmt;
7343 if (IsRotate && (HasROTL || HasROTR)) {
7345 MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt, LExtOp0,
7351 MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt, RExtOp0,
7358 MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, LHSShiftAmt, RHSShiftAmt,
7364 MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
7376 struct ByteProvider {
7381 unsigned ByteOffset = 0;
7383 ByteProvider() =
default;
7386 return ByteProvider(
Load, ByteOffset);
7389 static ByteProvider getConstantZero() {
return ByteProvider(
nullptr, 0); }
7391 bool isConstantZero()
const {
return !
Load; }
7392 bool isMemory()
const {
return Load; }
7394 bool operator==(
const ByteProvider &Other)
const {
7400 :
Load(
Load), ByteOffset(ByteOffset) {}
7418 bool Root =
false) {
7423 if (!Root && !
Op.hasOneUse())
7426 assert(
Op.getValueType().isScalarInteger() &&
"can't handle other types");
7431 assert(
Index < ByteWidth &&
"invalid index requested");
7434 switch (
Op.getOpcode()) {
7443 if (
LHS->isConstantZero())
7445 if (
RHS->isConstantZero())
7450 auto ShiftOp = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
7454 uint64_t BitShift = ShiftOp->getZExtValue();
7455 if (BitShift % 8 != 0)
7459 return Index < ByteShift
7460 ? ByteProvider::getConstantZero()
7469 if (NarrowBitWidth % 8 != 0)
7471 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
7473 if (
Index >= NarrowByteWidth)
7483 auto L = cast<LoadSDNode>(
Op.getNode());
7484 if (!L->isSimple() || L->isIndexed())
7487 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
7488 if (NarrowBitWidth % 8 != 0)
7490 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
7492 if (
Index >= NarrowByteWidth)
7515 int64_t FirstOffset) {
7521 bool BigEndian =
true, LittleEndian =
true;
7522 for (
unsigned i = 0; i <
Width; i++) {
7523 int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
7526 if (!BigEndian && !LittleEndian)
7530 assert((BigEndian != LittleEndian) &&
"It should be either big endian or"
7536 switch (
Value.getOpcode()) {
7579 EVT MemVT =
N->getMemoryVT();
7581 !
N->isSimple() ||
N->isIndexed())
7587 while (
auto *
Store = dyn_cast<StoreSDNode>(Chain)) {
7591 if (
Store->getMemoryVT() != MemVT || !
Store->isSimple() ||
7595 Chain =
Store->getChain();
7598 if (Stores.
size() < 2)
7603 unsigned NumStores = Stores.
size();
7604 unsigned NarrowNumBits =
N->getMemoryVT().getScalarSizeInBits();
7605 unsigned WideNumBits = NumStores * NarrowNumBits;
7617 for (
auto Store : Stores) {
7628 isa<ConstantSDNode>(WideVal.
getOperand(1))) {
7636 if (ShiftAmtC % NarrowNumBits != 0)
7639 Offset = ShiftAmtC / NarrowNumBits;
7646 SourceValue = WideVal;
7653 SourceValue = WideVal;
7661 int64_t ByteOffsetFromBase = 0;
7664 else if (!
Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
7668 if (ByteOffsetFromBase < FirstOffset) {
7670 FirstOffset = ByteOffsetFromBase;
7674 if (Offset < 0 || Offset >= NumStores || OffsetMap[
Offset] !=
INT64_MAX)
7676 OffsetMap[
Offset] = ByteOffsetFromBase;
7680 assert(FirstStore &&
"First store must be set");
7692 auto checkOffsets = [&](
bool MatchLittleEndian) {
7693 if (MatchLittleEndian) {
7694 for (
unsigned i = 0; i != NumStores; ++i)
7695 if (OffsetMap[i] != i * (NarrowNumBits / 8) + FirstOffset)
7698 for (
unsigned i = 0, j = NumStores - 1; i != NumStores; ++i, --j)
7699 if (OffsetMap[j] != i * (NarrowNumBits / 8) + FirstOffset)
7706 bool NeedBswap =
false;
7707 bool NeedRotate =
false;
7710 if (NarrowNumBits == 8 && checkOffsets(Layout.
isBigEndian()))
7712 else if (NumStores == 2 && checkOffsets(Layout.
isBigEndian()))
7721 "Unexpected store value to merge");
7730 }
else if (NeedRotate) {
7731 assert(WideNumBits % 2 == 0 &&
"Unexpected type for rotate");
7777 "Can only match load combining against OR nodes");
7780 EVT VT =
N->getValueType(0);
7786 auto MemoryByteOffset = [&] (ByteProvider
P) {
7787 assert(
P.isMemory() &&
"Must be a memory byte provider");
7788 unsigned LoadBitWidth =
P.Load->getMemoryVT().getSizeInBits();
7789 assert(LoadBitWidth % 8 == 0 &&
7790 "can only analyze providers for individual bytes not bit");
7791 unsigned LoadByteWidth = LoadBitWidth / 8;
7792 return IsBigEndianTarget
7807 unsigned ZeroExtendedBytes = 0;
7808 for (
int i = ByteWidth - 1; i >= 0; --i) {
7813 if (
P->isConstantZero()) {
7816 if (++ZeroExtendedBytes != (ByteWidth -
static_cast<unsigned>(i)))
7820 assert(
P->isMemory() &&
"provenance should either be memory or zero");
7825 "Must be enforced by calculateByteProvider");
7832 else if (Chain != LChain)
7837 int64_t ByteOffsetFromBase = 0;
7840 else if (!
Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
7844 ByteOffsetFromBase += MemoryByteOffset(*
P);
7845 ByteOffsets[i] = ByteOffsetFromBase;
7848 if (ByteOffsetFromBase < FirstOffset) {
7849 FirstByteProvider =
P;
7850 FirstOffset = ByteOffsetFromBase;
7855 assert(!Loads.
empty() &&
"All the bytes of the value must be loaded from "
7856 "memory, so there must be at least one load which produces the value");
7857 assert(
Base &&
"Base address of the accessed memory location must be set");
7860 bool NeedsZext = ZeroExtendedBytes > 0;
7871 if (LegalOperations &&
7879 makeArrayRef(ByteOffsets).drop_back(ZeroExtendedBytes), FirstOffset);
7883 assert(FirstByteProvider &&
"must be set");
7887 if (MemoryByteOffset(*FirstByteProvider) != 0)
7889 LoadSDNode *FirstLoad = FirstByteProvider->Load;
7896 bool NeedsBswap = IsBigEndianTarget != *IsBigEndian;
7903 if (NeedsBswap && (LegalOperations || NeedsZext) &&
7909 if (NeedsBswap && NeedsZext && LegalOperations &&
7937 SDLoc(
N), LegalOperations))
7965 EVT VT =
N->getValueType(0);
7987 M =
And.getOperand(XorIdx ? 0 : 1);
7993 if (!matchAndXor(N0, 0, N1) && !matchAndXor(N0, 1, N1) &&
7994 !matchAndXor(N1, 0, N0) && !matchAndXor(N1, 1, N0))
8000 if (isa<ConstantSDNode>(
M.getNode()))
8068 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
8080 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
8091 isSetCCEquivalent(N0,
LHS,
RHS, CC,
true)) {
8093 LHS.getValueType());
8094 if (!LegalOperations ||
8112 CombineTo(
N, SetCC);
8114 recursivelyDeleteUnusedNodes(N0.
getNode());
8138 if (isOneUseSetCC(N01) || isOneUseSetCC(N00)) {
8143 return DAG.
getNode(NewOpcode,
DL, VT, N00, N01);
8150 if (isa<ConstantSDNode>(N01) || isa<ConstantSDNode>(N00)) {
8155 return DAG.
getNode(NewOpcode,
DL, VT, N00, N01);
8179 AddToWorklist(NotX.
getNode());
8187 if (XorC && ShiftC) {
8193 Ones = N0Opcode ==
ISD::SHL ? Ones.
shl(ShiftAmt) : Ones.
lshr(ShiftAmt);
8210 SDValue A0 =
A.getOperand(0), A1 =
A.getOperand(1);
8212 if ((A0 == S && A1 == S0) || (A1 == S && A0 == S0))
8249 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
8253 if (
SDValue MM = unfoldMaskedMerge(
N))
8276 unsigned LogicOpcode = LogicOp.
getOpcode();
8282 unsigned ShiftOpcode = Shift->
getOpcode();
8285 assert(C1Node &&
"Expected a shift with constant operand");
8288 const APInt *&ShiftAmtVal) {
8302 if (ShiftAmtVal->getBitWidth() != C1Val.
getBitWidth())
8315 if (matchFirstShift(LogicOp.
getOperand(0),
X, C0Val))
8317 else if (matchFirstShift(LogicOp.
getOperand(1),
X, C0Val))
8329 return DAG.
getNode(LogicOpcode,
DL, VT, NewShift1, NewShift2);
8362 switch (
LHS.getOpcode()) {
8387 isa<ConstantSDNode>(BinOpLHSVal.
getOperand(1));
8391 if (!IsShiftByConstant && !IsCopyOrSelect)
8394 if (IsCopyOrSelect &&
N->hasOneUse())
8399 EVT VT =
N->getValueType(0);
8402 assert(isa<ConstantSDNode>(NewRHS) &&
"Folding was not successful!");
8406 return DAG.
getNode(
LHS.getOpcode(),
DL, VT, NewShift, NewRHS);
8414 EVT TruncVT =
N->getValueType(0);
8415 if (
N->hasOneUse() &&
N->getOperand(0).hasOneUse() &&
8417 SDValue N01 =
N->getOperand(0).getOperand(1);
8420 SDValue N00 =
N->getOperand(0).getOperand(0);
8423 AddToWorklist(Trunc00.
getNode());
8424 AddToWorklist(Trunc01.
getNode());
8436 EVT VT =
N->getValueType(0);
8451 bool OutOfRange =
false;
8453 OutOfRange |=
C->getAPIntValue().uge(Bitsize);
8461 return DAG.
getNode(
N->getOpcode(), dl, VT, N0, Amt);
8466 if (RotAmtC && RotAmtC->getAPIntValue() == 8 &&
8478 return DAG.
getNode(
N->getOpcode(), dl, VT, N0, NewOp1);
8488 bool SameSide = (
N->getOpcode() == NextOp);
8491 CombineOp, dl, ShiftVT, {N1, N0.getOperand(1)})) {
8494 ISD::SREM, dl, ShiftVT, {CombinedShift, BitsizeC});
8542 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
8566 return (c1 + c2).uge(OpSizeInBits);
8576 return (c1 + c2).ult(OpSizeInBits);
8604 return c2.
uge(OpSizeInBits - InnerBitwidth) &&
8605 (c1 + c2).uge(OpSizeInBits);
8617 return c2.
uge(OpSizeInBits - InnerBitwidth) &&
8618 (c1 + c2).ult(OpSizeInBits);
8652 AddToWorklist(NewSHL.
getNode());
8664 uint64_t C1 = N0C1->getZExtValue();
8684 if (N0C1->getAPIntValue().ult(OpSizeInBits)) {
8695 Mask.lshrInPlace(c1 - c2);
8727 AddToWorklist(Shl0.
getNode());
8728 AddToWorklist(Shl1.
getNode());
8742 if (
SDValue NewSHL = visitShiftByConstant(
N))
8749 const APInt &C1 = NC1->getAPIntValue();
8759 APInt NewStep = C0 << ShlVal;
8774 "SRL or SRA node is required here!");
8785 SDValue ShiftOperand =
N->getOperand(0);
8796 if (!IsSignExt && !IsZeroExt)
8804 unsigned ActiveBits = IsSignExt
8805 ?
Constant->getAPIntValue().getMinSignedBits()
8806 :
Constant->getAPIntValue().getActiveBits();
8807 if (ActiveBits > NarrowVTSize)
8824 "Cannot have a multiply node with two different operand types.");
8835 if (ShiftAmt != NarrowVTSize)
8876 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
8888 if (!LegalOperations ||
8911 APInt Sum = c1 + c2;
8923 "Expected matchBinaryPredicate to return one element for "
8927 ShiftValue = ShiftValues[0];
8955 if ((ShiftAmt > 0) &&
8967 N->getValueType(0), Trunc);
9024 if (LargeShift->getAPIntValue() == TruncBits) {
9044 if (
SDValue NewSRA = visitShiftByConstant(
N))
9053 if (
SDValue NarrowLoad = reduceLoadWidth(
N))
9077 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
9093 return (c1 + c2).uge(OpSizeInBits);
9103 return (c1 + c2).ult(OpSizeInBits);
9125 if (c1 + OpSizeInBits == InnerShiftSize) {
9127 if (c1 + c2 >= InnerShiftSize)
9137 c1 + c2 < InnerShiftSize) {
9158 AddToWorklist(
Mask.getNode());
9178 AddToWorklist(SmallShift.
getNode());
9222 AddToWorklist(
Op.getNode());
9244 if (
SDValue NewSRL = visitShiftByConstant(
N))
9248 if (
SDValue NarrowLoad = reduceLoadWidth(
N))
9268 if (
N->hasOneUse()) {
9289 EVT VT =
N->getValueType(0);
9301 return IsFSHL ? N0 : N1;
9303 auto IsUndefOrZero = [](
SDValue V) {
9320 return IsFSHL ? N0 : N1;
9326 if (IsUndefOrZero(N0))
9330 if (IsUndefOrZero(N1))
9342 auto *
LHS = dyn_cast<LoadSDNode>(N0);
9343 auto *
RHS = dyn_cast<LoadSDNode>(N1);
9344 if (
LHS &&
RHS &&
LHS->isSimple() &&
RHS->isSimple() &&
9345 LHS->getAddressSpace() ==
RHS->getAddressSpace() &&
9355 RHS->getAddressSpace(), NewAlign,
9356 RHS->getMemOperand()->getFlags(), &Fast) &&
9360 AddToWorklist(NewPtr.
getNode());
9362 VT,
DL,
RHS->getChain(), NewPtr,
9363 RHS->getPointerInfo().getWithOffset(PtrOff), NewAlign,
9364 RHS->getMemOperand()->getFlags(),
RHS->getAAInfo());
9366 WorklistRemover DeadNodes(*
this);
9392 if (N0 == N1 && hasOperation(RotOpc, VT))
9454 EVT VT =
N->getValueType(0);
9474 EVT VT =
N->getValueType(0);
9498 EVT VT =
N->getValueType(0);
9511 EVT VT =
N->getValueType(0);
9528 EVT VT =
N->getValueType(0);
9538 EVT VT =
N->getValueType(0);
9555 EVT VT =
N->getValueType(0);
9565 EVT VT =
N->getValueType(0);
9579 EVT VT =
LHS.getValueType();
9591 if (!(
LHS == True &&
RHS == False) && !(
LHS == False &&
RHS == True))
9643 EVT VT =
N->getValueType(0);
9645 VT !=
Cond.getOperand(0).getValueType())
9675 EVT VT =
N->getValueType(0);
9676 EVT CondVT =
Cond.getValueType();
9682 auto *C1 = dyn_cast<ConstantSDNode>(N1);
9683 auto *C2 = dyn_cast<ConstantSDNode>(N2);
9691 if (CondVT ==
MVT::i1 && !LegalOperations) {
9692 if (C1->
isZero() && C2->isOne()) {
9699 if (C1->
isZero() && C2->isAllOnes()) {
9706 if (C1->
isOne() && C2->isZero()) {
9724 const APInt &C1Val = C1->getAPIntValue();
9725 const APInt &C2Val = C2->getAPIntValue();
9726 if (C1Val - 1 == C2Val) {
9732 if (C1Val + 1 == C2Val) {
9768 C1->
isZero() && C2->isOne()) {
9781 "Expected a (v)select");
9783 SDValue T =
N->getOperand(1),
F =
N->getOperand(2);
9784 EVT VT =
N->getValueType(0);
9817 EVT VT =
N->getValueType(0);
9875 EVT VT =
N->getValueType(0);
9883 if (
SDValue V = foldSelectOfConstants(
N))
9890 if (SimplifySelectOps(
N, N1, N2))
9902 bool normalizeToSequence =
9911 if (normalizeToSequence || !InnerSelect.
use_empty())
9913 InnerSelect, N2, Flags);
9916 recursivelyDeleteUnusedNodes(InnerSelect.
getNode());
9923 Cond1, N1, N2, Flags);
9924 if (normalizeToSequence || !InnerSelect.
use_empty())
9926 InnerSelect, Flags);
9929 recursivelyDeleteUnusedNodes(InnerSelect.
getNode());
9939 if (!normalizeToSequence) {
9945 if (
SDValue Combined = visitANDLike(N0, N1_0,
N)) {
9958 if (!normalizeToSequence) {
9964 if (
SDValue Combined = visitORLike(N0, N2_0,
N))
10000 auto *
C = dyn_cast<ConstantSDNode>(N2.
getOperand(1));
10001 auto *NotC = dyn_cast<ConstantSDNode>(Cond1);
10002 if (
C && NotC &&
C->getAPIntValue() == ~NotC->getAPIntValue()) {
10022 (!LegalOperations &&
10033 if (
SDValue NewSel = SimplifySelect(
DL, N0, N1, N2))
10038 if (
SDValue BinOp = foldSelectOfBinops(
N))
10051 EVT VT =
N->getValueType(0);
10059 if (
LHS->getNumOperands() != 2 ||
RHS->getNumOperands() != 2)
10068 for (
int i = 0; i < NumElems / 2; ++i) {
10069 if (
Cond->getOperand(i)->isUndef())
10072 if (BottomHalf ==
nullptr)
10073 BottomHalf = cast<ConstantSDNode>(
Cond.getOperand(i));
10074 else if (
Cond->getOperand(i).getNode() != BottomHalf)
10080 for (
int i = NumElems / 2; i < NumElems; ++i) {
10081 if (
Cond->getOperand(i)->isUndef())
10084 if (TopHalf ==
nullptr)
10085 TopHalf = cast<ConstantSDNode>(
Cond.getOperand(i));
10086 else if (
Cond->getOperand(i).getNode() != TopHalf)
10090 assert(TopHalf && BottomHalf &&
10091 "One half of the selector was all UNDEFs and the other was all the "
10092 "same value. This should have been addressed before this function.");
10095 BottomHalf->
isZero() ?
RHS->getOperand(0) :
LHS->getOperand(0),
10096 TopHalf->
isZero() ?
RHS->getOperand(1) :
LHS->getOperand(1));
10109 BasePtr = SplatVal;
10193 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
10197 Value.getValueType().isInteger() &&
10198 (!isa<ConstantSDNode>(
Value) ||
10199 !cast<ConstantSDNode>(
Value)->isOpaque())) {
10200 APInt TruncDemandedBits =
10225 Value.getOperand(0).getValueType());
10247 return CombineTo(
N, PassThru, MGT->
getChain());
10285 return CombineTo(
N, NewLd, NewLd.
getValue(1));
10289 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
10301 EVT VT =
N->getValueType(0);
10302 if (!
Cond.hasOneUse() ||
Cond.getScalarValueSizeInBits() != 1 ||
10311 bool AllAddOne =
true;
10312 bool AllSubOne =
true;
10314 for (
unsigned i = 0; i != Elts; ++i) {
10322 const APInt &C1 = cast<ConstantSDNode>(N1Elt)->getAPIntValue();
10323 const APInt &C2 = cast<ConstantSDNode>(N2Elt)->getAPIntValue();
10333 if (AllAddOne || AllSubOne) {
10364 EVT VT =
N->getValueType(0);
10385 bool isAbs =
false;
10404 AddToWorklist(Shift.
getNode());
10405 AddToWorklist(
Add.getNode());
10432 EVT NarrowVT =
LHS.getValueType();
10440 SetCCWidth != 1 && SetCCWidth < WideWidth &&
10483 (OpLHS == CondLHS || OpRHS == CondLHS))
10486 if (OpRHS.getOpcode() == CondRHS.getOpcode() &&
10489 CondLHS == OpLHS) {
10494 return Cond->getAPIntValue() == ~
Op->getAPIntValue();
10516 if (Other &&
Other.getNumOperands() == 2) {
10533 if (OpLHS ==
LHS) {
10549 return (!
Op && !
Cond) ||
10551 Cond->getAPIntValue() == (-
Op->getAPIntValue() - 1));
10583 if (SimplifySelectOps(
N, N1, N2))
10603 if (
SDValue V = foldVSelectOfConstants(
N))
10628 AddToWorklist(
SCC.getNode());
10631 if (!SCCC->isZero())
10635 }
else if (
SCC->isUndef()) {
10643 SCC.getOperand(1), N2, N3,
SCC.getOperand(2));
10650 if (SimplifySelectOps(
N, N2, N3))
10654 return SimplifySelectCC(
SDLoc(
N), N0, N1, N2, N3, CC);
10662 N->hasOneUse() &&
N->use_begin()->getOpcode() ==
ISD::BRCOND;
10665 EVT VT =
N->getValueType(0);
10682 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
10685 bool Updated =
false;
10697 return True || False;
10701 if (!IsAlwaysTrueOrFalse(
Cond, N1C)) {
10719 SDLoc(
N), !PreferSetCC);
10727 SDValue NewSetCC = rebuildSetCC(Combined);
10761 if (!
N.hasOneUse())
10764 if (!isa<LoadSDNode>(
N))
10789 unsigned Opcode =
N->getOpcode();
10791 EVT VT =
N->getValueType(0);
10796 "Expected EXTEND dag node in input!");
10832 unsigned Opcode =
N->getOpcode();
10834 EVT VT =
N->getValueType(0);
10840 &&
"Expected EXTEND dag node in input!");
10845 if (isa<ConstantSDNode>(N0))
10846 return DAG.
getNode(Opcode,
DL, VT, N0);
10854 if (isa<ConstantSDNode>(Op1) && isa<ConstantSDNode>(Op2) &&
10865 unsigned FoldOpc = Opcode;
10893 for (
unsigned i = 0; i != NumElts; ++i) {
10895 if (
Op.isUndef()) {
10903 APInt C = cast<ConstantSDNode>(
Op)->getAPIntValue().zextOrTrunc(EVTBits);
10921 bool HasCopyToRegUses =
false;
10929 if (UI.getUse().getResNo() != N0.
getResNo())
10938 for (
unsigned i = 0; i != 2; ++i) {
10942 if (!isa<ConstantSDNode>(UseOp))
10956 HasCopyToRegUses =
true;
10959 if (HasCopyToRegUses) {
10960 bool BothLiveOut =
false;
10965 BothLiveOut =
true;
10972 return ExtendNodes.
size();
10982 for (
SDNode *SetCC : SetCCs) {
10985 for (
unsigned j = 0; j != 2; ++j) {
10987 if (SOp == OrigLoad)
11001 EVT DstVT =
N->getValueType(0);
11006 "Unexpected node type (not an extend)!");
11044 EVT SplitSrcVT = SrcVT;
11045 EVT SplitDstVT = DstVT;
11058 const unsigned NumSplits =
11065 for (
unsigned Idx = 0; Idx < NumSplits; Idx++) {
11066 const unsigned Offset = Idx * Stride;
11084 AddToWorklist(NewChain.
getNode());
11086 CombineTo(
N, NewValue);
11092 ExtendSetCCUses(SetCCs, N0, NewValue, (
ISD::NodeType)
N->getOpcode());
11093 CombineTo(N0.
getNode(), Trunc, NewChain);
11101 EVT VT =
N->getValueType(0);
11102 EVT OrigVT =
N->getOperand(0).getValueType();
11125 EVT MemVT =
Load->getMemoryVT();
11146 Load->getChain(),
Load->getBasePtr(),
11147 Load->getMemoryVT(),
Load->getMemOperand());
11164 Load->getValueType(0), ExtLoad);
11169 recursivelyDeleteUnusedNodes(N0.
getNode());
11178 SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(
SDNode *Cast) {
11179 unsigned CastOpcode = Cast->
getOpcode();
11183 "Unexpected opcode for vector select narrowing/widening");
11213 CastA = DAG.
getNode(CastOpcode,
DL, VT, A);
11223 bool LegalOperations,
SDNode *
N,
11234 if ((LegalOperations || !LN0->
isSimple() ||
11245 Combiner.recursivelyDeleteUnusedNodes(LN0);
11259 ((LegalOperations || VT.
isVector() ||
11260 !cast<LoadSDNode>(N0)->isSimple()) &&
11264 bool DoXform =
true;
11277 Combiner.ExtendSetCCUses(SetCCs, N0, ExtLoad, ExtOpc);
11281 if (NoReplaceTrunc) {
11283 Combiner.recursivelyDeleteUnusedNodes(LN0);
11321 bool LegalOperations) {
11333 EVT VT =
N->getValueType(0);
11334 EVT XVT =
X.getValueType();
11350 return DAG.
getNode(ShiftOpcode,
DL, VT, NotX, ShiftAmount);
11364 EVT VT =
N->getValueType(0);
11371 if (VT.
isVector() && !LegalOperations &&
11390 if (SVT == MatchingVecType) {
11406 auto IsFreeToExtend = [&](
SDValue V) {
11414 cast<LoadSDNode>(V)->isSimple() &&
11424 if (UI.getUse().getResNo() != 0 ||
User == N0.
getNode())
11429 if (
User->getOpcode() != ExtOpcode ||
User->getValueType(0) != VT)
11435 if (IsFreeToExtend(N00) && IsFreeToExtend(N01)) {
11438 return DAG.
getSetCC(
DL, VT, Ext0, Ext1, CC);
11455 SDValue ExtTrueVal = (SetCCWidth == 1)
11459 if (
SDValue SCC = SimplifySelectCC(
DL, N00, N01, ExtTrueVal, Zero, CC,
true))
11471 return DAG.
getSelect(
DL, VT, SetCC, ExtTrueVal, Zero);
11480 EVT VT =
N->getValueType(0);
11496 if (NarrowLoad.getNode() != N0.
getNode()) {
11497 CombineTo(N0.
getNode(), NarrowLoad);
11499 AddToWorklist(oye);
11507 unsigned OpBits =
Op.getScalarValueSizeInBits();
11512 if (OpBits == DestBits) {
11515 if (NumSignBits > DestBits-MidBits)
11517 }
else if (OpBits < DestBits) {
11520 if (NumSignBits > OpBits-MidBits)
11525 if (NumSignBits > OpBits-MidBits)
11532 if (OpBits < DestBits)
11534 else if (OpBits > DestBits)
11554 if (
SDValue ExtLoad = CombineExtLoad(
N))
11585 bool NoReplaceTruncAnd = !N0.
hasOneUse();
11589 if (NoReplaceTruncAnd) {
11592 CombineTo(N0.
getNode(), TruncAnd);
11594 if (NoReplaceTrunc) {
11599 CombineTo(LN00, Trunc, ExtLoad.
getValue(1));
11617 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
11648 if (NewXor.getNode() == N0.
getNode()) {
11675 Op =
N->getOperand(0);
11681 N.getValueType().getScalarType() !=
MVT::i1 ||
11682 cast<CondCodeSDNode>(
N.getOperand(2))->get() !=
ISD::SETNE)
11726 EVT VT =
N->getValueType(0);
11743 APInt TruncatedBits =
11745 APInt(
Op.getScalarValueSizeInBits(), 0) :
11760 if (NarrowLoad.getNode() != N0.
getNode()) {
11761 CombineTo(N0.
getNode(), NarrowLoad);
11763 AddToWorklist(oye);
11778 AddToWorklist(
Op.getNode());
11782 return ZExtOrTrunc;
11788 AddToWorklist(
Op.getNode());
11826 if (
SDValue ExtLoad = CombineExtLoad(
N))
11842 bool DoXform =
true;
11846 auto *AndC = cast<ConstantSDNode>(N0.
getOperand(1));
11849 if (isAndLoadExtLoad(AndC, LN00, LoadResultTy, ExtVT))
11866 bool NoReplaceTruncAnd = !N0.
hasOneUse();
11870 if (NoReplaceTruncAnd) {
11873 CombineTo(N0.
getNode(), TruncAnd);
11875 if (NoReplaceTrunc) {
11880 CombineTo(LN00, Trunc, ExtLoad.
getValue(1));
11889 if (
SDValue ZExtLoad = CombineZExtLogicopShiftLoad(
N))
11902 if (!LegalOperations && VT.
isVector() &&
11940 cast<CondCodeSDNode>(N0.
getOperand(2))->get(),
true))
11956 if (cast<ConstantSDNode>(ShAmt)->getAPIntValue().ugt(KnownZeroBits))
11971 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
11985 EVT VT =
N->getValueType(0);
12003 if (NarrowLoad.getNode() != N0.
getNode()) {
12004 CombineTo(N0.
getNode(), NarrowLoad);
12006 AddToWorklist(oye);
12043 bool DoXform =
true;
12056 CombineTo(
N, ExtLoad);
12057 if (NoReplaceTrunc) {
12059 recursivelyDeleteUnusedNodes(LN0);
12063 CombineTo(LN0, Trunc, ExtLoad.
getValue(1));
12077 if (!LegalOperations || TLI.
isLoadExtLegal(ExtType, VT, MemVT)) {
12081 CombineTo(
N, ExtLoad);
12083 recursivelyDeleteUnusedNodes(LN0);
12094 if (VT.
isVector() && !LegalOperations) {
12107 cast<CondCodeSDNode>(N0.
getOperand(2))->get());
12116 cast<CondCodeSDNode>(N0.
getOperand(2))->get());
12125 cast<CondCodeSDNode>(N0.
getOperand(2))->get(),
true))
12139 unsigned Opcode =
N->getOpcode();
12142 EVT AssertVT = cast<VTSDNode>(N1)->getVT();
12146 AssertVT == cast<VTSDNode>(N0.
getOperand(1))->getVT())
12157 EVT BigA_AssertVT = cast<VTSDNode>(BigA.
getOperand(1))->getVT();
12159 "Asserting zero/sign-extended bits to a type larger than the "
12160 "truncated destination does not provide information");
12163 EVT MinAssertVT = AssertVT.
bitsLT(BigA_AssertVT) ? AssertVT : BigA_AssertVT;
12177 EVT BigA_AssertVT = cast<VTSDNode>(BigA.
getOperand(1))->getVT();
12179 "Asserting zero/sign-extended bits to a type larger than the "
12180 "truncated destination does not provide information");
12182 if (AssertVT.
bitsLT(BigA_AssertVT)) {
12196 Align AL = cast<AssertAlignSDNode>(
N)->getAlign();
12201 if (
auto *AAN = dyn_cast<AssertAlignSDNode>(N0))
12213 unsigned AlignShift =
Log2(
AL);
12218 if (LHSAlignShift >= AlignShift || RHSAlignShift >= AlignShift) {
12219 if (LHSAlignShift < AlignShift)
12221 if (RHSAlignShift < AlignShift)
12236 unsigned Opc =
N->getOpcode();
12240 EVT VT =
N->getValueType(0);
12250 unsigned ShAmt = 0;
12255 bool HasShiftedOffset =
false;
12260 ExtVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
12268 auto *LN = dyn_cast<LoadSDNode>(N0);
12269 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
12275 uint64_t MemoryWidth = LN->getMemoryVT().getScalarSizeInBits();
12276 if (MemoryWidth <= ShAmt)
12287 LN->getExtensionType() != ExtType)
12291 auto AndC = dyn_cast<ConstantSDNode>(
N->getOperand(1));
12296 unsigned ActiveBits = 0;
12297 if (
Mask.isMask()) {
12298 ActiveBits =
Mask.countTrailingOnes();
12299 }
else if (
Mask.isShiftedMask()) {
12300 ShAmt =
Mask.countTrailingZeros();
12303 HasShiftedOffset =
true;
12323 if (!
SRL.hasOneUse())
12328 auto *LN = dyn_cast<LoadSDNode>(
SRL.getOperand(0));
12329 auto *SRL1C = dyn_cast<ConstantSDNode>(
SRL.getOperand(1));
12336 ShAmt = SRL1C->getZExtValue();
12337 uint64_t MemoryWidth = LN->getMemoryVT().getSizeInBits();
12338 if (ShAmt >= MemoryWidth)
12365 isa<ConstantSDNode>(
Mask->getOperand(1))) {
12366 const APInt& ShiftMask =
Mask->getConstantOperandAPInt(1);
12367 if (ShiftMask.
isMask()) {
12377 N0 =
SRL.getOperand(0);
12385 unsigned ShLeftAmt = 0;
12389 ShLeftAmt = N01->getZExtValue();
12395 if (!isa<LoadSDNode>(N0))
12402 !isLegalNarrowLdSt(LN0, ExtType, ExtVT, ShAmt))
12405 auto AdjustBigEndianShift = [&](
unsigned ShAmt) {
12406 unsigned LVTStoreBits =
12409 return LVTStoreBits - EVTStoreBits - ShAmt;
12414 unsigned PtrAdjustmentInBits =
12417 uint64_t PtrOff = PtrAdjustmentInBits / 8;
12425 AddToWorklist(NewPtr.
getNode());
12439 WorklistRemover DeadNodes(*
this);
12444 if (ShLeftAmt != 0) {
12459 if (HasShiftedOffset) {
12476 EVT VT =
N->getValueType(0);
12477 EVT ExtVT = cast<VTSDNode>(N1)->getVT();
12506 if ((N00Bits <= ExtVTBits ||
12524 if ((N00Bits == ExtVTBits ||
12525 (!IsZext && (N00Bits < ExtVTBits ||
12527 (!LegalOperations ||
12552 if (
SDValue NarrowLoad = reduceLoadWidth(
N))
12559 if (
auto *ShAmt = dyn_cast<ConstantSDNode>(N0.
getOperand(1)))
12560 if (ShAmt->getAPIntValue().ule(VTBits - ExtVTBits)) {
12564 if (((VTBits - ExtVTBits) - ShAmt->getZExtValue()) < InSignBits)
12576 ExtVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
12577 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple() &&
12585 CombineTo(
N, ExtLoad);
12587 AddToWorklist(ExtLoad.
getNode());
12594 ExtVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
12595 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple()) &&
12602 CombineTo(
N, ExtLoad);
12610 if (ExtVT == Ld->getMemoryVT() && N0.
hasOneUse() &&
12614 VT,
SDLoc(
N), Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(),
12615 Ld->getMask(), Ld->getPassThru(), ExtVT, Ld->getMemOperand(),
12616 Ld->getAddressingMode(),
ISD::SEXTLOAD, Ld->isExpandingLoad());
12617 CombineTo(
N, ExtMaskedLoad);
12624 if (
auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
12626 ExtVT == GN0->getMemoryVT() &&
12628 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
12629 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
12635 CombineTo(
N, ExtLoad);
12637 AddToWorklist(ExtLoad.
getNode());
12654 EVT VT =
N->getValueType(0);
12671 EVT VT =
N->getValueType(0);
12686 if (
C.getNode() !=
N)
12723 EVT TrTy =
N->getValueType(0);
12727 auto NewEltCnt = EltCnt * SizeRatio;
12733 if (isa<ConstantSDNode>(EltNo) &&
isTypeLegal(NVT)) {
12734 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
12735 int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1));
12770 AddToWorklist(Amt.
getNode());
12807 if (BuildVectEltTy == TruncVecEltTy) {
12811 unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts;
12813 assert((BuildVecNumElts % TruncVecNumElts) == 0 &&
12814 "Invalid number of elements");
12817 for (
unsigned i = 0,
e = BuildVecNumElts; i !=
e; i += TruncEltOffset)
12839 if (
SDValue Reduced = reduceLoadWidth(
N))
12863 unsigned NumDefs = 0;
12867 if (!
X.isUndef()) {
12878 X.getValueType().getVectorElementCount()));
12884 if (NumDefs == 1) {
12885 assert(V.
getNode() &&
"The single defined operand is empty!");
12887 for (
unsigned i = 0,
e = VTs.
size(); i !=
e; ++i) {
12893 AddToWorklist(
NV.getNode());
12908 (!LegalOperations ||
12953 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
12967 if (!LegalOperations && N0.
hasOneUse() &&
12985 if (!LegalOperations && N0.
hasOneUse() &&
13021 !LD1->hasOneUse() || !LD2->hasOneUse() ||
13022 LD1->getAddressSpace() != LD2->getAddressSpace())
13025 bool LD1Fast =
false;
13026 EVT LD1VT = LD1->getValueType(0);
13031 *LD1->getMemOperand(), &LD1Fast) && LD1Fast)
13032 return DAG.
getLoad(VT,
SDLoc(
N), LD1->getChain(), LD1->getBasePtr(),
13033 LD1->getPointerInfo(), LD1->getAlign());
13048 EVT VT =
N->getValueType(0);
13088 NumFPLogicOpsConv++;
13099 EVT VT =
N->getValueType(0);
13115 cast<BuildVectorSDNode>(N0)->isConstant())
13116 return ConstantFoldBITCASTofBUILD_VECTOR(N0.
getNode(),
13124 if (!LegalOperations ||
13130 if (
C.getNode() !=
N)
13150 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple()) ||
13185 AddToWorklist(NewConv.
getNode());
13195 AddToWorklist(FlipBit.
getNode());
13202 AddToWorklist(
Hi.getNode());
13204 AddToWorklist(FlipBit.
getNode());
13208 AddToWorklist(FlipBits.
getNode());
13238 AddToWorklist(
X.getNode());
13242 if (OrigXWidth < VTWidth) {
13244 AddToWorklist(
X.getNode());
13245 }
else if (OrigXWidth > VTWidth) {
13250 X.getValueType(),
X,
13252 X.getValueType()));
13253 AddToWorklist(
X.getNode());
13255 AddToWorklist(
X.getNode());
13261 AddToWorklist(Cst.
getNode());
13263 AddToWorklist(
X.getNode());
13265 AddToWorklist(XorResult.
getNode());
13269 SDLoc(XorResult)));
13270 AddToWorklist(XorResult64.
getNode());
13274 AddToWorklist(FlipBit.
getNode());
13277 AddToWorklist(FlipBits.
getNode());
13283 AddToWorklist(
X.getNode());
13288 AddToWorklist(Cst.
getNode());
13296 if (
SDValue CombineLD = CombineConsecutiveLoads(N0.
getNode(), VT))
13311 auto PeekThroughBitcast = [&](
SDValue Op) {
13313 Op.getOperand(0).getValueType() == VT)
13333 for (
int i = 0; i != MaskScale; ++i)
13334 NewMask.
push_back(M < 0 ? -1 : M * MaskScale + i);
13339 return LegalShuffle;
13346 EVT VT =
N->getValueType(0);
13347 return CombineConsecutiveLoads(
N, VT);
13362 ConstantFoldBITCASTofBUILD_VECTOR(
SDNode *BV,
EVT DstEltVT) {
13366 if (SrcEltVT == DstEltVT)
return SDValue(BV, 0);
13373 if (SrcBitSize == DstBitSize) {
13378 if (
Op.getValueType() != SrcEltVT)
13381 AddToWorklist(Ops.
back().getNode());
13395 BV = ConstantFoldBITCASTofBUILD_VECTOR(BV, IntVT).getNode();
13403 SDNode *Tmp = ConstantFoldBITCASTofBUILD_VECTOR(BV, TmpVT).getNode();
13406 return ConstantFoldBITCASTofBUILD_VECTOR(Tmp, DstEltVT);
13414 auto *BVN = cast<BuildVectorSDNode>(BV);
13420 if (!BVN->getConstantRawBits(IsLE, DstBitSize, RawBits, UndefElements))
13425 for (
unsigned I = 0,
E = RawBits.
size();
I !=
E; ++
I) {
13426 if (UndefElements[
I])
13442 N->getFlags().hasAllowContract();
13447 return Options.NoInfsFPMath ||
N.getNode()->getFlags().hasNoInfs();
13454 EVT VT =
N->getValueType(0);
13460 bool HasFMAD = (LegalOperations && TLI.
isFMADLegal(DAG,
N));
13468 if (!HasFMAD && !HasFMA)
13471 bool CanReassociate =
13472 Options.UnsafeFPMath ||
N->getFlags().hasAllowReassociation();
13474 Options.UnsafeFPMath || HasFMAD);
13476 if (!AllowFusionGlobally && !
N->getFlags().hasAllowContract())
13487 unsigned Opcode =
N.getOpcode();
13496 return AllowFusionGlobally ||
N->getFlags().hasAllowContract();
13522 if (CanReassociate && isFusedOp(N0) &&
13527 }
else if (CanReassociate && isFusedOp(N1) &&
13539 return DAG.
getNode(PreferredFusedOpcode, SL, VT, A,
B, CDE);
13550 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
13564 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
13577 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X,
Y,
13578 DAG.
getNode(PreferredFusedOpcode, SL, VT,
13583 if (isFusedOp(N0)) {
13607 DAG.
getNode(PreferredFusedOpcode, SL, VT,
13613 if (isFusedOp(N00)) {
13627 if (isFusedOp(N1)) {
13648 if (isFusedOp(N10)) {
13668 EVT VT =
N->getValueType(0);
13673 bool HasFMAD = (LegalOperations && TLI.
isFMADLegal(DAG,
N));
13681 if (!HasFMAD && !HasFMA)
13686 Options.UnsafeFPMath || HasFMAD);
13689 if (!AllowFusionGlobally && !
N->getFlags().hasAllowContract())
13705 return AllowFusionGlobally ||
N->getFlags().hasAllowContract();
13721 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
13733 if (
SDValue V = tryToFoldXSubYZ(N0, N1))
13736 if (
SDValue V = tryToFoldXYSubZ(N0, N1))
13740 if (
SDValue V = tryToFoldXYSubZ(N0, N1))
13743 if (
SDValue V = tryToFoldXSubYZ(N0, N1))
13752 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
13766 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
13782 PreferredFusedOpcode, SL, VT,
13804 DAG.
getNode(PreferredFusedOpcode, SL, VT,
13827 DAG.
getNode(PreferredFusedOpcode, SL, VT,
13836 return Options.UnsafeFPMath ||
N->getFlags().hasAllowReassociation();
13845 unsigned Opcode =
N.getOpcode();
13851 bool CanFuse =
Options.UnsafeFPMath ||
N->getFlags().hasAllowContract();
13854 if (CanFuse && isFusedOp(N0) &&
13855 isContractableAndReassociableFMUL(N0.
getOperand(2)) &&
13859 DAG.
getNode(PreferredFusedOpcode, SL, VT,
13867 if (CanFuse && isFusedOp(N1) &&
13868 isContractableAndReassociableFMUL(N1.
getOperand(2)) &&
13873 PreferredFusedOpcode, SL, VT,
13875 DAG.
getNode(PreferredFusedOpcode, SL, VT,
13881 if (isFusedOp(N0) && N0->
hasOneUse()) {
13885 if (isContractableAndReassociableFMUL(N020) &&
13891 PreferredFusedOpcode, SL, VT,
13907 if (isFusedOp(N00)) {
13909 if (isContractableAndReassociableFMUL(N002) &&
13913 PreferredFusedOpcode, SL, VT,
13917 PreferredFusedOpcode, SL, VT,
13930 if (isContractableAndReassociableFMUL(N120) &&
13936 PreferredFusedOpcode, SL, VT,
13938 DAG.
getNode(PreferredFusedOpcode, SL, VT,
13956 if (isContractableAndReassociableFMUL(N102) &&
13962 PreferredFusedOpcode, SL, VT,
13966 DAG.
getNode(PreferredFusedOpcode, SL, VT,
13980 SDValue DAGCombiner::visitFMULForFMADistributiveCombine(
SDNode *
N) {
13983 EVT VT =
N->getValueType(0);
14004 bool HasFMAD =
Options.UnsafeFPMath &&
14008 if (!HasFMAD && !HasFMA)
14020 if (
C->isExactlyValue(+1.0))
14021 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
14023 if (
C->isExactlyValue(-1.0))
14024 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
14043 if (C0->isExactlyValue(+1.0))
14044 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
14047 if (C0->isExactlyValue(-1.0))
14048 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
14053 if (C1->isExactlyValue(+1.0))
14054 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
14056 if (C1->isExactlyValue(-1.0))
14057 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
14077 EVT VT =
N->getValueType(0);
14091 if (N0CFP && !N1CFP)
14096 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
14101 if (N1C && N1C->
isZero())
14105 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
14111 N1, DAG, LegalOperations, ForCodeSize))
14117 N0, DAG, LegalOperations, ForCodeSize))
14124 return C &&
C->isExactlyValue(-2.0);
14128 if (isFMulNegTwo(N0)) {
14134 if (isFMulNegTwo(N1)) {
14177 if (CFP01 && !CFP00 && N0.
getOperand(0) == N1) {
14198 if (CFP11 && !CFP10 && N1.
getOperand(0) == N0) {
14246 if (
SDValue Fused = visitFADDForFMACombine(
N)) {
14247 AddToWorklist(Fused.getNode());
14257 EVT VT =
N->getValueType(0);
14258 EVT ChainVT =
N->getValueType(1);
14265 N1, DAG, LegalOperations, ForCodeSize)) {
14267 {Chain, N0, NegN1});
14273 N0, DAG, LegalOperations, ForCodeSize)) {
14275 {Chain, N1, NegN0});
14285 EVT VT =
N->getValueType(0);
14300 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
14303 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
14307 if (N1CFP && N1CFP->
isZero()) {
14321 if (N0CFP && N0CFP->
isZero()) {
14356 if (
SDValue Fused = visitFSUBForFMACombine(
N)) {
14357 AddToWorklist(Fused.getNode());
14368 EVT VT =
N->getValueType(0);
14388 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
14391 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
14440 if (NegN0 && NegN1 &&
14455 auto TrueOpnd = dyn_cast<ConstantFPSDNode>(
Select.getOperand(1));
14456 auto FalseOpnd = dyn_cast<ConstantFPSDNode>(
Select.getOperand(2));
14458 if (TrueOpnd && FalseOpnd &&
14460 isa<ConstantFPSDNode>(
Cond.getOperand(1)) &&
14461 cast<ConstantFPSDNode>(
Cond.getOperand(1))->isExactlyValue(0.0)) {
14479 if (TrueOpnd->isExactlyValue(-1.0) && FalseOpnd->isExactlyValue(1.0) &&
14483 if (TrueOpnd->isExactlyValue(1.0) && FalseOpnd->isExactlyValue(-1.0))
14492 if (
SDValue Fused = visitFMULForFMADistributiveCombine(
N)) {
14493 AddToWorklist(Fused.getNode());
14506 EVT VT =
N->getValueType(0);
14512 bool UnsafeFPMath =
14513 Options.UnsafeFPMath ||
N->getFlags().hasAllowReassociation();
14516 if (isa<ConstantFPSDNode>(N0) &&
14517 isa<ConstantFPSDNode>(N1) &&
14518 isa<ConstantFPSDNode>(N2)) {
14531 if (NegN0 && NegN1 &&
14536 if (UnsafeFPMath) {
14537 if (N0CFP && N0CFP->
isZero())
14539 if (N1CFP && N1CFP->
isZero())
14553 if (UnsafeFPMath) {
14580 AddToWorklist(RHSNeg.
getNode());
14594 if (UnsafeFPMath) {
14596 if (N1CFP && N0 == N2) {
14614 SDValue(
N, 0), DAG, LegalOperations, ForCodeSize))
14636 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
14647 unsigned NumElts = 1;
14648 EVT VT =
N->getValueType(0);
14652 if (!MinUses || (N1->
use_size() * NumElts) < MinUses)
14658 for (
auto *U : N1->
uses()) {
14659 if (U->getOpcode() ==
ISD::FDIV && U->getOperand(1) == N1) {
14661 if (U->getOperand(1).getOpcode() ==
ISD::FSQRT &&
14662 U->getOperand(0) == U->getOperand(1).getOperand(0) &&
14663 U->getFlags().hasAllowReassociation() &&
14664 U->getFlags().hasNoSignedZeros())
14669 if (UnsafeMath || U->getFlags().hasAllowReciprocal())
14676 if ((
Users.size() * NumElts) < MinUses)
14684 for (
auto *U :
Users) {
14686 if (Dividend != FPOne) {
14688 Reciprocal, Flags);
14689 CombineTo(U, NewNode);
14690 }
else if (U != Reciprocal.
getNode()) {
14693 CombineTo(U, Reciprocal);
14702 EVT VT =
N->getValueType(0);
14717 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
14720 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
14728 if (
auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1)) {
14736 (!LegalOperations ||
14785 A =
Y.getOperand(0);
14794 if (
SDValue Rsqrt = buildRsqrtEstimate(AAZ, Flags))
14798 recursivelyDeleteUnusedNodes(AAZ.
getNode());
14806 AddToWorklist(Div.
getNode());
14814 if (
SDValue RV = BuildDivEstimate(N0, N1, Flags))
14833 if (NegN0 && NegN1 &&
14844 EVT VT =
N->getValueType(0);
14855 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
14880 return buildSqrtEstimate(N0, Flags);
14893 if (N1VT == N1Op0VT)
14915 EVT VT =
N->getValueType(0);
14969 EVT VT =
N->getValueType(0);
14999 if (ExponentIs025 || ExponentIs075) {
15044 EVT VT =
N->getValueType(0);
15065 EVT VT =
N->getValueType(0);
15075 (!LegalOperations ||
15117 EVT VT =
N->getValueType(0);
15127 (!LegalOperations ||
15157 EVT VT =
N->getValueType(0);
15163 EVT SrcVT = Src.getValueType();
15178 unsigned ActualSize =
std::min(InputSize, OutputSize);
15198 EVT VT =
N->getValueType(0);
15213 EVT VT =
N->getValueType(0);
15230 EVT VT =
N->getValueType(0);
15242 const bool NIsTrunc =
N->getConstantOperandVal(1) == 1;
15271 AddToWorklist(Tmp.
getNode());
15276 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
15284 EVT VT =
N->getValueType(0);
15287 if (
N->hasOneUse() &&
15305 if (
In.getValueType() == VT)
return In;
15320 CombineTo(
N, ExtLoad);
15329 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
15337 EVT VT =
N->getValueType(0);
15348 EVT VT =
N->getValueType(0);
15372 EVT VT =
N->getValueType(0);
15383 EVT VT =
N->getValueType(0);
15400 N->getFlags().hasNoSignedZeros()) && N0.
hasOneUse()) {
15405 if (
SDValue Cast = foldSignChangeInBitcast(
N))
15414 EVT VT =
N->getValueType(0);
15416 unsigned Opc =
N->getOpcode();
15438 return PropagatesNaN ?
N->getOperand(1) :
N->getOperand(0);
15448 return N->getOperand(1);
15455 return N->getOperand(0);
15464 EVT VT =
N->getValueType(0);
15472 return N->getOperand(0);
15479 if (
SDValue Cast = foldSignChangeInBitcast(
N))
15517 if (
SDValue NewN1 = rebuildSetCC(N1))
15519 ChainHandle.getValue(), NewN1, N2);
15528 (
N.getOperand(0).hasOneUse() &&
15529 N.getOperand(0).getOpcode() ==
ISD::SRL))) {
15532 N =
N.getOperand(0);
15558 const APInt &AndConst = cast<ConstantSDNode>(AndOp1)->getAPIntValue();
15561 cast<ConstantSDNode>(Op1)->getAPIntValue() == AndConst.
logBase2()) {
15581 SDValue Tmp = visitXOR(
N.getNode());
15588 N = XORHandle.getValue();
15600 bool Equal =
false;
15605 Op0 =
N->getOperand(0);
15606 Op1 =
N->getOperand(1);
15610 EVT SetCCVT =
N.getValueType();
15626 SDValue CondLHS =
N->getOperand(2), CondRHS =
N->getOperand(3);
15651 bool &IsLoad,
bool &IsMasked,
SDValue &Ptr,
15654 if (
LD->isIndexed())
15656 EVT VT =
LD->getMemoryVT();
15659 Ptr =
LD->getBasePtr();
15661 if (
ST->isIndexed())
15663 EVT VT =
ST->getMemoryVT();
15666 Ptr =
ST->getBasePtr();
15669 if (
LD->isIndexed())
15671 EVT VT =
LD->getMemoryVT();
15675 Ptr =
LD->getBasePtr();
15678 if (
ST->isIndexed())
15680 EVT VT =
ST->getMemoryVT();
15684 Ptr =
ST->getBasePtr();
15698 bool DAGCombiner::CombineToPreIndexedLoadStore(
SDNode *
N) {
15702 bool IsLoad =
true;
15703 bool IsMasked =
false;
15725 bool Swapped =
false;
15726 if (isa<ConstantSDNode>(BasePtr)) {
15745 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
15750 SDValue Val = IsMasked ? cast<MaskedStoreSDNode>(
N)->getValue()
15751 : cast<StoreSDNode>(
N)->getValue();
15754 if (Val == BasePtr)
15771 if (isa<ConstantSDNode>(
Offset))
15773 UE =
BasePtr.getNode()->use_end();
15791 if (!isa<ConstantSDNode>(Op1)) {
15809 bool RealUse =
false;
15844 Result.getNode()->dump(&DAG);
dbgs() <<
'\n');
15845 WorklistRemover DeadNodes(*
this);
15854 deleteAndRecombine(
N);
15860 for (
unsigned i = 0,
e = OtherUses.
size(); i !=
e; ++i) {
15861 unsigned OffsetIdx = 1;
15862 if (OtherUses[i]->getOperand(OffsetIdx).getNode() ==
BasePtr.getNode())
15864 assert(OtherUses[i]->getOperand(!OffsetIdx).getNode() ==
15865 BasePtr.getNode() &&
"Expected BasePtr operand");
15878 auto *CN = cast<ConstantSDNode>(OtherUses[i]->getOperand(OffsetIdx));
15879 const APInt &Offset0 = CN->getAPIntValue();
15880 const APInt &Offset1 = cast<ConstantSDNode>(
Offset)->getAPIntValue();
15881 int X0 = (OtherUses[i]->getOpcode() ==
ISD::SUB && OffsetIdx == 1) ? -1 : 1;
15882 int Y0 = (OtherUses[i]->getOpcode() ==
ISD::SUB && OffsetIdx == 0) ? -1 : 1;
15888 APInt CNV = Offset0;
15889 if (X0 < 0) CNV = -CNV;
15890 if (X1 * Y0 * Y1 < 0) CNV = CNV + Offset1;
15891 else CNV = CNV - Offset1;
15903 deleteAndRecombine(OtherUses[i]);
15908 deleteAndRecombine(Ptr.
getNode());
15909 AddToWorklist(
Result.getNode());
15930 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
15934 for (
SDNode *
Use : BasePtr.getNode()->uses()) {
15939 if (isa<MemSDNode>(
Use)) {
15940 bool IsLoad =
true;
15941 bool IsMasked =
false;
15944 IsMasked, OtherPtr, TLI)) {
15964 bool &IsMasked,
SDValue &Ptr,
15970 IsMasked, Ptr, TLI) ||
16003 bool DAGCombiner::CombineToPostIndexedLoadStore(
SDNode *
N) {
16007 bool IsLoad =
true;
16008 bool IsMasked =
false;
16029 ++PostIndexedNodes;
16032 dbgs() <<
"\nWith: ";
Result.getNode()->dump(&DAG);
16034 WorklistRemover DeadNodes(*
this);
16043 deleteAndRecombine(
N);
16047 Result.getValue(IsLoad ? 1 : 0));
16048 deleteAndRecombine(
Op);
16063 !cast<ConstantSDNode>(Inc)->isOpaque()) &&
16064 "Cannot split out indexing using opaque target constants");
16081 Val =
ST->getValue();
16083 EVT STMemType =
ST->getMemoryVT();
16084 if (STType == STMemType)
16106 EVT LDMemType =
LD->getMemoryVT();
16107 EVT LDType =
LD->getValueType(0);
16109 "Attempting to extend value of non-matching type");
16110 if (LDType == LDMemType)
16113 switch (
LD->getExtensionType()) {
16137 if (!
ST || !
ST->isSimple())
16140 EVT LDType =
LD->getValueType(0);
16141 EVT LDMemType =
LD->getMemoryVT();
16142 EVT STMemType =
ST->getMemoryVT();
16143 EVT STType =
ST->getValue().getValueType();
16166 if (!BasePtrST.equalBaseIndex(BasePtrLD, DAG,
Offset))
16185 STCoversLD = (
Offset == 0) && LdMemSize == StMemSize;
16191 if (
LD->isIndexed()) {
16196 SDValue Idx = SplitIndexingFromLoad(
LD);
16197 SDValue Ops[] = {Val, Idx, Chain};
16198 return CombineTo(
LD, Ops, 3);
16200 return CombineTo(
LD, Val, Chain);
16207 if (
Offset == 0 && LDType == STType && STMemType == LDMemType) {
16210 return ReplaceLd(
LD,
ST->getValue(), Chain);
16220 return ReplaceLd(
LD, Val, Chain);
16225 if (
LD->getBasePtr().isUndef() ||
Offset != 0)
16231 if (!getTruncatedStoreValue(
ST, Val))
16235 if (STMemType != LDMemType) {
16243 if (!extendLoadedValueToExtension(
LD, Val))
16245 return ReplaceLd(
LD, Val, Chain);
16250 deleteAndRecombine(Val.
getNode());
16263 if (
LD->isSimple()) {
16266 if (!
N->hasAnyUseOfValue(0)) {
16276 WorklistRemover DeadNodes(*
this);
16278 AddUsersToWorklist(Chain.
getNode());
16279 if (
N->use_empty())
16280 deleteAndRecombine(
N);
16294 if (!
N->hasAnyUseOfValue(0) && (CanSplitIdx || !
N->hasAnyUseOfValue(1))) {
16297 if (
N->hasAnyUseOfValue(1) && CanSplitIdx) {
16298 Index = SplitIndexingFromLoad(
LD);
16301 AddUsersToWorklist(
N);
16305 dbgs() <<
"\nWith: ";
Undef.getNode()->dump(&DAG);
16306 dbgs() <<
" and 2 other values\n");
16307 WorklistRemover DeadNodes(*
this);
16311 deleteAndRecombine(
N);
16319 if (
auto V = ForwardStoreValueToDirectLoad(
LD))
16325 if (*Alignment >
LD->getAlign() &&
16326 isAligned(*Alignment,
LD->getSrcValueOffset())) {
16328 LD->getExtensionType(),
SDLoc(
N),
LD->getValueType(0), Chain, Ptr,
16329 LD->getPointerInfo(),
LD->getMemoryVT(), *Alignment,
16330 LD->getMemOperand()->getFlags(),
LD->getAAInfo());
16338 if (
LD->isUnindexed()) {
16340 SDValue BetterChain = FindBetterChain(
LD, Chain);
16343 if (Chain != BetterChain) {
16349 BetterChain, Ptr,
LD->getMemOperand());
16352 LD->getValueType(0),
16353 BetterChain, Ptr,
LD->getMemoryVT(),
16354 LD->getMemOperand());
16362 return CombineTo(
N, ReplLoad.
getValue(0), Token);
16367 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
16372 if (SliceUpLoad(
N))
16392 struct LoadedSlice {
16396 bool ForCodeSize =
false;
16399 unsigned Loads = 0;
16400 unsigned Truncates = 0;
16401 unsigned CrossRegisterBanksCopies = 0;
16402 unsigned ZExts = 0;
16403 unsigned Shift = 0;
16405 explicit Cost(
bool ForCodeSize) : ForCodeSize(ForCodeSize) {}
16408 Cost(
const LoadedSlice &
LS,
bool ForCodeSize)
16409 : ForCodeSize(ForCodeSize), Loads(1) {
16410 EVT TruncType =
LS.Inst->getValueType(0);
16411 EVT LoadedType =
LS.getLoadedType();
16412 if (TruncType != LoadedType &&
16413 !
LS.DAG->getTargetLoweringInfo().isZExtFree(LoadedType, TruncType))
16421 void addSliceGain(
const LoadedSlice &
LS) {
16425 LS.Inst->getValueType(0)))
16431 if (
LS.canMergeExpensiveCrossRegisterBankCopy())
16432 ++CrossRegisterBanksCopies;
16436 Loads +=
RHS.Loads;
16437 Truncates +=
RHS.Truncates;
16438 CrossRegisterBanksCopies +=
RHS.CrossRegisterBanksCopies;
16439 ZExts +=
RHS.ZExts;
16440 Shift +=
RHS.Shift;
16445 return Loads ==
RHS.Loads && Truncates ==
RHS.Truncates &&
16446 CrossRegisterBanksCopies ==
RHS.CrossRegisterBanksCopies &&
16447 ZExts ==
RHS.ZExts && Shift ==
RHS.Shift;
16455 unsigned ExpensiveOpsLHS = Loads + CrossRegisterBanksCopies;
16456 unsigned ExpensiveOpsRHS =
RHS.Loads +
RHS.CrossRegisterBanksCopies;
16459 if (!ForCodeSize && ExpensiveOpsLHS != ExpensiveOpsRHS)
16460 return ExpensiveOpsLHS < ExpensiveOpsRHS;
16461 return (Truncates + ZExts + Shift + ExpensiveOpsLHS) <
16462 (
RHS.Truncates +
RHS.ZExts +
RHS.Shift + ExpensiveOpsRHS);
16488 : Inst(Inst), Origin(Origin), Shift(Shift), DAG(DAG) {}
16493 APInt getUsedBits()
const {
16498 assert(Origin &&
"No original load to compare against.");
16500 assert(Inst &&
"This slice is not bound to an instruction");
16502 "Extracted slice is bigger than the whole type!");
16504 UsedBits.setAllBits();
16505 UsedBits = UsedBits.zext(
BitWidth);
16506 UsedBits <<= Shift;
16511 unsigned getLoadedSize()
const {
16512 unsigned SliceSize = getUsedBits().countPopulation();
16513 assert(!(SliceSize & 0x7) &&
"Size is not a multiple of a byte.");
16514 return SliceSize / 8;
16519 EVT getLoadedType()
const {
16520 assert(DAG &&
"Missing context");
16535 bool isLegal()
const {
16537 if (!Origin || !Inst || !DAG)
16547 EVT SliceType = getLoadedType();
16571 if (TruncateType != SliceType &&
16581 uint64_t getOffsetFromBase()
const {
16582 assert(DAG &&
"Missing context.");
16584 assert(!(Shift & 0x7) &&
"Shifts not aligned on Bytes are not supported.");
16588 "The size of the original loaded type is not a multiple of a"
16593 "Invalid shift amount for given loaded size");
16606 assert(Inst && Origin &&
"Unable to replace a non-existing slice.");
16608 SDValue BaseAddr = OldBaseAddr;
16610 int64_t
Offset =
static_cast<int64_t
>(getOffsetFromBase());
16611 assert(
Offset >= 0 &&
"Offset too big to fit in int64_t!");
16621 EVT SliceType = getLoadedType();
16631 if (SliceType != FinalType)
16641 bool canMergeExpensiveCrossRegisterBankCopy()
const {
16647 assert(DAG &&
"Missing context");
16649 EVT ResVT =
Use->getValueType(0);
16654 Use->getOperand(0)->isDivergent());
16663 if (!
TRI ||
TRI->getCommonSubClass(ArgRC, ResRC))
16668 bool IsFast =
false;
16709 const LoadedSlice &Second) {
16710 assert(First.Origin == Second.Origin && First.Origin &&
16711 "Unable to match different memory origins.");
16712 APInt UsedBits = First.getUsedBits();
16713 assert((UsedBits & Second.getUsedBits()) == 0 &&
16714 "Slices are not supposed to overlap.");
16715 UsedBits |= Second.getUsedBits();
16724 LoadedSlice::Cost &GlobalLSCost) {
16725 unsigned NumberOfSlices = LoadedSlices.
size();
16727 if (NumberOfSlices < 2)
16732 llvm::sort(LoadedSlices, [](
const LoadedSlice &
LHS,
const LoadedSlice &
RHS) {
16733 assert(
LHS.Origin ==
RHS.Origin &&
"Different bases not implemented.");
16734 return LHS.getOffsetFromBase() <
RHS.getOffsetFromBase();
16736 const TargetLowering &TLI = LoadedSlices[0].DAG->getTargetLoweringInfo();
16739 const LoadedSlice *First =
nullptr;
16740 const LoadedSlice *Second =
nullptr;
16741 for (
unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice,
16744 Second = &LoadedSlices[CurrSlice];
16751 EVT LoadedType = First->getLoadedType();
16754 if (LoadedType != Second->getLoadedType())
16758 Align RequiredAlignment;
16759 if (!TLI.hasPairedLoad(LoadedType, RequiredAlignment)) {
16765 if (First->getAlign() < RequiredAlignment)
16772 assert(GlobalLSCost.Loads > 0 &&
"We save more loads than we created!");
16773 --GlobalLSCost.Loads;
16790 const APInt &UsedBits,
bool ForCodeSize) {
16791 unsigned NumberOfSlices = LoadedSlices.
size();
16793 return NumberOfSlices > 1;
16796 if (NumberOfSlices != 2)
16804 LoadedSlice::Cost OrigCost(ForCodeSize), GlobalSlicingCost(ForCodeSize);
16806 OrigCost.Loads = 1;
16807 for (
unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice) {
16808 const LoadedSlice &
LS = LoadedSlices[CurrSlice];
16810 LoadedSlice::Cost SliceCost(
LS, ForCodeSize);
16811 GlobalSlicingCost += SliceCost;
16815 OrigCost.addSliceGain(
LS);
16820 return OrigCost > GlobalSlicingCost;
16829 bool DAGCombiner::SliceUpLoad(
SDNode *
N) {
16835 !
LD->getValueType(0).isInteger())
16841 if (
LD->getValueType(0).isScalableVector())
16846 APInt UsedBits(
LD->getValueSizeInBits(0), 0);
16854 UI != UIEnd; ++UI) {
16856 if (UI.getUse().getResNo() != 0)
16860 unsigned Shift = 0;
16865 Shift =
User->getConstantOperandVal(1);
16878 unsigned Width =
User->getValueSizeInBits(0);
16883 LoadedSlice
LS(
User,
LD, Shift, &DAG);
16884 APInt CurrentUsedBits =
LS.getUsedBits();
16887 if ((CurrentUsedBits & UsedBits) != 0)
16890 UsedBits |= CurrentUsedBits;
16911 for (
const LoadedSlice &
LS : LoadedSlices) {
16913 CombineTo(
LS.Inst, SliceInst,
true);
16917 "It takes more than a zext to get to the loaded slice!!");
16924 AddToWorklist(Chain.
getNode());
16931 static std::pair<unsigned, unsigned>
16933 std::pair<unsigned, unsigned> Result(0, 0);
16943 if (
LD->getBasePtr() != Ptr)
return Result;
16956 if (NotMaskLZ & 7)
return Result;
16958 if (NotMaskTZ & 7)
return Result;
16959 if (NotMaskLZ == 64)
return Result;
16970 switch (MaskedBytes) {
16974 default:
return Result;
16979 if (NotMaskTZ && NotMaskTZ/8 % MaskedBytes)
return Result;
16993 Result.first = MaskedBytes;
16994 Result.second = NotMaskTZ/8;
17005 unsigned NumBytes = MaskInfo.first;
17006 unsigned ByteShift = MaskInfo.second;
17012 ByteShift*8, (ByteShift+NumBytes)*8);
17020 if (!
DC->isTypeLegal(VT))
17039 StOffset = ByteShift;
17065 if (!
ST->isSimple())
17076 unsigned Opc =
Value.getOpcode();
17084 std::pair<unsigned, unsigned> MaskedLoad;
17086 if (MaskedLoad.first)
17088 Value.getOperand(1),
ST,
this))
17093 if (MaskedLoad.first)
17095 Value.getOperand(0),
ST,
this))
17110 if (
LD->getBasePtr() != Ptr ||
17111 LD->getPointerInfo().getAddrSpace() !=
17112 ST->getPointerInfo().getAddrSpace())
17118 APInt Imm = cast<ConstantSDNode>(N1)->getAPIntValue();
17142 ShAmt = (((ShAmt + NewBW - 1) / NewBW) * NewBW) - NewBW;
17145 if ((Imm &
Mask) == Imm) {
17153 PtrOff = (
BitWidth + 7 - NewBW) / 8 - PtrOff;
17155 bool IsFast =
false;
17158 LD->getAddressSpace(), NewAlign,
17159 LD->getMemOperand()->getFlags(), &IsFast) ||
17167 LD->getPointerInfo().getWithOffset(PtrOff), NewAlign,
17168 LD->getMemOperand()->getFlags(),
LD->getAAInfo());
17174 ST->getPointerInfo().getWithOffset(PtrOff), NewAlign);
17176 AddToWorklist(NewPtr.
getNode());
17177 AddToWorklist(NewLD.
getNode());
17178 AddToWorklist(NewVal.
getNode());
17179 WorklistRemover DeadNodes(*
this);
17198 EVT VT =
LD->getMemoryVT();
17200 VT !=
ST->getMemoryVT() ||
17201 LD->isNonTemporal() ||
17202 ST->isNonTemporal() ||
17203 LD->getPointerInfo().getAddrSpace() != 0 ||
17204 ST->getPointerInfo().getAddrSpace() != 0)
17214 bool FastLD =
false, FastST =
false;
17221 *
LD->getMemOperand(), &FastLD) ||
17223 *
ST->getMemOperand(), &FastST) ||
17224 !FastLD || !FastST)
17229 LD->getPointerInfo(),
LD->getAlign());
17233 ST->getPointerInfo(),
ST->getAlign());
17235 AddToWorklist(NewLD.
getNode());
17236 AddToWorklist(NewST.
getNode());
17237 WorklistRemover DeadNodes(*
this);
17259 bool DAGCombiner::isMulAddWithConstProfitable(
SDNode *MulNode,
17272 if (
Use == MulNode)
17280 if (
Use->getOperand(0) == ConstNode)
17281 OtherOp =
Use->getOperand(1).getNode();
17283 OtherOp =
Use->getOperand(0).getNode();
17295 if (OtherOp == MulVar)
17323 unsigned NumStores) {
17326 SDLoc StoreDL(StoreNodes[0].MemNode);
17328 for (
unsigned i = 0; i < NumStores; ++i) {
17329 Visited.
insert(StoreNodes[i].MemNode);
17333 for (
unsigned i = 0; i < NumStores; ++i) {
17334 if (Visited.
insert(StoreNodes[i].MemNode->getChain().getNode()).second)
17335 Chains.
push_back(StoreNodes[i].MemNode->getChain());
17338 assert(Chains.
size() > 0 &&
"Chain should have generated a chain");
17342 bool DAGCombiner::mergeStoresOfConstantsOrVecElts(
17344 bool IsConstantSrc,
bool UseVector,
bool UseTrunc) {
17349 assert((!UseTrunc || !UseVector) &&
17350 "This optimization cannot emit a vector truncating store");
17353 SDLoc DL(StoreNodes[0].MemNode);
17356 unsigned SizeInBits = NumStores * ElementSizeBits;
17361 for (
unsigned I = 0;
I != NumStores; ++
I) {
17362 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[
I].MemNode);
17377 unsigned Elts = NumStores * NumMemElts;
17385 if (IsConstantSrc) {
17387 for (
unsigned I = 0;
I != NumStores; ++
I) {
17388 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[
I].MemNode);
17397 if (isa<ConstantFPSDNode>(Val)) {
17400 }
else if (
auto *
C = dyn_cast<ConstantSDNode>(Val))
17403 .zextOrTrunc(ElementSizeBits),
17413 DL, StoreTy, BuildVector);
17416 for (
unsigned i = 0; i < NumStores; ++i) {
17417 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
17451 assert(IsConstantSrc &&
"Merged vector elements should use vector store");
17453 APInt StoreInt(SizeInBits, 0);
17458 for (
unsigned i = 0; i < NumStores; ++i) {
17459 unsigned Idx = IsLE ? (NumStores - 1 - i) : i;
17460 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[Idx].MemNode);
17462 SDValue Val = St->getValue();
17464 StoreInt <<= ElementSizeBits;
17466 StoreInt |=
C->getAPIntValue()
17467 .zextOrTrunc(ElementSizeBits)
17468 .zextOrTrunc(SizeInBits);
17470 StoreInt |=
C->getValueAPF()
17472 .zextOrTrunc(ElementSizeBits)
17473 .zextOrTrunc(SizeInBits);
17487 SDValue NewChain = getMergeStoreChains(StoreNodes, NumStores);
17496 EVT LegalizedStoredValTy =
17498 unsigned LegalizedStoreSize = LegalizedStoredValTy.
getSizeInBits();
17501 DAG.
getConstant(
C->getAPIntValue().zextOrTrunc(LegalizedStoreSize),
DL,
17502 LegalizedStoredValTy);
17504 NewChain,
DL, ExtendedStoreVal, FirstInChain->
getBasePtr(),
17510 for (
unsigned i = 0; i < NumStores; ++i)
17511 CombineTo(StoreNodes[i].MemNode, NewStore);
17513 AddToWorklist(NewChain.
getNode());
17517 void DAGCombiner::getStoreMergeCandidates(
17524 if (!
BasePtr.getBase().getNode() ||
BasePtr.getBase().isUndef())
17528 StoreSource StoreSrc = getStoreSource(Val);
17536 auto *Ld = cast<LoadSDNode>(Val);
17538 LoadVT = Ld->getMemoryVT();
17540 if (MemVT != LoadVT)
17543 if (!Ld->hasNUsesOfValue(1, 0))
17547 if (!Ld->isSimple() || Ld->isIndexed())
17551 int64_t &
Offset) ->
bool {
17554 if (!
Other->isSimple() ||
Other->isIndexed())
17562 :
Other->getMemoryVT() != MemVT;
17563 switch (StoreSrc) {
17568 auto *OtherLd = dyn_cast<LoadSDNode>(OtherBC);
17572 if (LoadVT != OtherLd->getMemoryVT())
17575 if (!OtherLd->hasNUsesOfValue(1, 0))
17579 if (!OtherLd->isSimple() || OtherLd->isIndexed())
17582 if (cast<LoadSDNode>(Val)->isNonTemporal() != OtherLd->isNonTemporal())
17584 if (!(LBasePtr.equalBaseIndex(LPtr, DAG)))
17594 case StoreSource::Extract:
17596 if (
Other->isTruncatingStore())
17613 auto OverLimitInDependenceCheck = [&](
SDNode *StoreNode,
17614 SDNode *RootNode) ->
bool {
17615 auto RootCount = StoreRootCountMap.
find(StoreNode);
17616 return RootCount != StoreRootCountMap.
end() &&
17617 RootCount->second.first == RootNode &&
17623 if (UseIter.getOperandNo() != 0)
17625 if (
auto *OtherStore = dyn_cast<StoreSDNode>(*UseIter)) {
17628 if (CandidateMatch(OtherStore, Ptr, PtrDiff) &&
17629 !OverLimitInDependenceCheck(OtherStore, RootNode))
17630 StoreNodes.
push_back(MemOpLink(OtherStore, PtrDiff));
17652 unsigned NumNodesExplored = 0;
17653 const unsigned MaxSearchNodes = 1024;
17654 if (
auto *Ldn = dyn_cast<LoadSDNode>(RootNode)) {
17655 RootNode = Ldn->getChain().getNode();
17657 I !=
E && NumNodesExplored < MaxSearchNodes; ++
I, ++NumNodesExplored) {
17658 if (
I.getOperandNo() == 0 && isa<LoadSDNode>(*
I)) {
17659 for (
auto I2 = (*I)->use_begin(), E2 = (*I)->use_end(); I2 != E2; ++I2)
17660 TryToAddCandidate(I2);
17663 if (
I.getOperandNo() == 0 && isa<StoreSDNode>(*
I)) {
17664 TryToAddCandidate(
I);
17669 I !=
E && NumNodesExplored < MaxSearchNodes; ++
I, ++NumNodesExplored)
17670 TryToAddCandidate(
I);
17679 bool DAGCombiner::checkMergeStoreCandidatesForDependencies(
17695 while (!Worklist.
empty()) {
17697 if (!Visited.
insert(
N).second)
17706 unsigned int Max = 1024 + Visited.
size();
17708 for (
unsigned i = 0; i < NumStores; ++i) {
17709 SDNode *
N = StoreNodes[i].MemNode;
17721 for (
unsigned j = 1; j <
N->getNumOperands(); ++j)
17722 Worklist.
push_back(
N->getOperand(j).getNode());
17725 for (
unsigned i = 0; i < NumStores; ++i)
17731 if (Visited.
size() >= Max) {
17732 auto &RootCount = StoreRootCountMap[StoreNodes[i].MemNode];
17733 if (RootCount.first == RootNode)
17734 RootCount.second++;
17736 RootCount = {RootNode, 1};
17745 int64_t ElementSizeBytes)
const {
17748 size_t StartIdx = 0;
17749 while ((StartIdx + 1 < StoreNodes.
size()) &&
17750 StoreNodes[StartIdx].OffsetFromBase + ElementSizeBytes !=
17751 StoreNodes[StartIdx + 1].OffsetFromBase)
17755 if (StartIdx + 1 >= StoreNodes.
size())
17764 unsigned NumConsecutiveStores = 1;
17765 int64_t StartAddress = StoreNodes[0].OffsetFromBase;
17768 for (
unsigned i = 1,
e = StoreNodes.
size(); i <
e; ++i) {
17769 int64_t CurrAddress = StoreNodes[i].OffsetFromBase;
17770 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
17772 NumConsecutiveStores = i + 1;
17774 if (NumConsecutiveStores > 1)
17775 return NumConsecutiveStores;
17783 bool DAGCombiner::tryStoreMergeOfConstants(
17785 EVT MemVT,
SDNode *RootNode,
bool AllowVectors) {
17790 bool MadeChange =
false;
17793 while (NumConsecutiveStores >= 2) {
17796 unsigned FirstStoreAlign = FirstInChain->
getAlignment();
17797 unsigned LastLegalType = 1;
17798 unsigned LastLegalVectorType = 1;
17799 bool LastIntegerTrunc =
false;
17800 bool NonZero =
false;
17801 unsigned FirstZeroAfterNonZero = NumConsecutiveStores;
17802 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
17803 StoreSDNode *
ST = cast<StoreSDNode>(StoreNodes[i].MemNode);
17805 bool IsElementZero =
false;
17807 IsElementZero =
C->isZero();
17809 IsElementZero =
C->getConstantFPValue()->isNullValue();
17810 if (IsElementZero) {
17811 if (NonZero && FirstZeroAfterNonZero == NumConsecutiveStores)
17812 FirstZeroAfterNonZero = i;
17814 NonZero |= !IsElementZero;
17817 unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
17819 bool IsFast =
false;
17831 LastIntegerTrunc =
false;
17832 LastLegalType = i + 1;
17836 EVT LegalizedStoredValTy =
17844 LastIntegerTrunc =
true;
17845 LastLegalType = i + 1;
17856 unsigned Elts = (i + 1) * NumMemElts;
17863 LastLegalVectorType = i + 1;
17867 bool UseVector = (LastLegalVectorType > LastLegalType) && AllowVectors;
17868 unsigned NumElem = (UseVector) ? LastLegalVectorType : LastLegalType;
17869 bool UseTrunc = LastIntegerTrunc && !UseVector;
17881 unsigned NumSkip = 1;
17882 while ((NumSkip < NumConsecutiveStores) &&
17883 (NumSkip < FirstZeroAfterNonZero) &&
17884 (StoreNodes[NumSkip].MemNode->getAlignment() <= FirstStoreAlign))
17888 NumConsecutiveStores -= NumSkip;
17893 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumElem,
17896 NumConsecutiveStores -= NumElem;
17900 MadeChange |= mergeStoresOfConstantsOrVecElts(StoreNodes, MemVT, NumElem,
17902 UseVector, UseTrunc);
17906 NumConsecutiveStores -= NumElem;
17911 bool DAGCombiner::tryStoreMergeOfExtracts(
17917 bool MadeChange =
false;
17920 while (NumConsecutiveStores >= 2) {
17923 unsigned FirstStoreAlign = FirstInChain->
getAlignment();
17924 unsigned NumStoresToMerge = 1;
17925 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
17927 unsigned Elts = (i + 1) * NumMemElts;
17929 bool IsFast =
false;
17940 NumStoresToMerge = i + 1;
17945 if (NumStoresToMerge < 2) {
17952 unsigned NumSkip = 1;
17953 while ((NumSkip < NumConsecutiveStores) &&
17954 (StoreNodes[NumSkip].MemNode->getAlignment() <= FirstStoreAlign))
17958 NumConsecutiveStores -= NumSkip;
17963 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumStoresToMerge,
17966 StoreNodes.
begin() + NumStoresToMerge);
17967 NumConsecutiveStores -= NumStoresToMerge;
17971 MadeChange |= mergeStoresOfConstantsOrVecElts(
17972 StoreNodes, MemVT, NumStoresToMerge,
false,
17975 StoreNodes.
erase(StoreNodes.
begin(), StoreNodes.
begin() + NumStoresToMerge);
17976 NumConsecutiveStores -= NumStoresToMerge;
17982 unsigned NumConsecutiveStores,
EVT MemVT,
17983 SDNode *RootNode,
bool AllowVectors,
17984 bool IsNonTemporalStore,
17985 bool IsNonTemporalLoad) {
17990 bool MadeChange =
false;
17999 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
18000 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
18006 int64_t LdOffset = 0;
18007 if (LdBasePtr.getBase().getNode()) {
18009 if (!LdBasePtr.equalBaseIndex(LdPtr, DAG, LdOffset))
18017 LoadNodes.
push_back(MemOpLink(Ld, LdOffset));
18020 while (NumConsecutiveStores >= 2 && LoadNodes.
size() >= 2) {
18021 Align RequiredAlignment;
18022 bool NeedRotate =
false;
18023 if (LoadNodes.
size() == 2) {
18027 StoreNodes[0].MemNode->getAlign() >= RequiredAlignment) {
18033 int64_t Offset0 = LoadNodes[0].OffsetFromBase;
18034 int64_t Offset1 = LoadNodes[1].OffsetFromBase;
18036 if (Offset0 - Offset1 == ElementSizeBytes &&
18046 LoadSDNode *FirstLoad = cast<LoadSDNode>(LoadNodes[0].MemNode);
18052 unsigned LastConsecutiveLoad = 1;
18055 unsigned LastLegalVectorType = 1;
18056 unsigned LastLegalIntegerType = 1;
18057 bool isDereferenceable =
true;
18058 bool DoIntegerTruncate =
false;
18059 int64_t StartAddress = LoadNodes[0].OffsetFromBase;
18061 for (
unsigned i = 1; i < LoadNodes.
size(); ++i) {
18063 if (LoadNodes[i].MemNode->getChain() != LoadChain)
18066 int64_t CurrAddress = LoadNodes[i].OffsetFromBase;
18067 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
18069 LastConsecutiveLoad = i;
18071 if (isDereferenceable && !LoadNodes[i].MemNode->isDereferenceable())
18072 isDereferenceable =
false;
18075 unsigned Elts = (i + 1) * NumMemElts;
18082 bool IsFastSt =
false;
18083 bool IsFastLd =
false;
18097 LastLegalVectorType = i + 1;
18101 unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
18112 LastLegalIntegerType = i + 1;
18113 DoIntegerTruncate =
false;
18130 LastLegalIntegerType = i + 1;
18131 DoIntegerTruncate =
true;
18139 LastLegalVectorType > LastLegalIntegerType && AllowVectors;
18140 unsigned LastLegalType =
18141 std::max(LastLegalVectorType, LastLegalIntegerType);
18145 unsigned NumElem =
std::min(NumConsecutiveStores, LastConsecutiveLoad + 1);
18146 NumElem =
std::min(LastLegalType, NumElem);
18157 unsigned NumSkip = 1;
18158 while ((NumSkip < LoadNodes.
size()) &&
18159 (LoadNodes[NumSkip].MemNode->getAlign() <= FirstLoadAlign) &&
18160 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign))
18164 NumConsecutiveStores -= NumSkip;
18169 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumElem,
18173 NumConsecutiveStores -= NumElem;
18182 unsigned Elts = NumElem * NumMemElts;
18185 unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
18189 SDLoc LoadDL(LoadNodes[0].MemNode);
18190 SDLoc StoreDL(StoreNodes[0].MemNode);
18195 SDValue NewStoreChain = getMergeStoreChains(StoreNodes, NumElem);
18196 AddToWorklist(NewStoreChain.
getNode());
18201 if (IsNonTemporalLoad)
18209 if (UseVectorTy || !DoIntegerTruncate) {
18215 unsigned LoadWidth = ElementSizeBytes * 8 * 2;
18217 "Unexpected type for rotate-able load pair");
18224 NewStoreChain, StoreDL, StoreOp, FirstInChain->
getBasePtr(),
18232 FirstLoadAlign, LdMMOFlags);
18234 NewStoreChain, StoreDL, NewLoad, FirstInChain->
getBasePtr(),
18240 for (
unsigned i = 0; i < NumElem; ++i) {
18241 LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode);
18248 for (
unsigned i = 0; i < NumElem; ++i) {
18249 SDValue Val = StoreNodes[i].MemNode->getOperand(1);
18250 CombineTo(StoreNodes[i].MemNode, NewStore);
18252 recursivelyDeleteUnusedNodes(Val.
getNode());
18258 NumConsecutiveStores -= NumElem;
18263 bool DAGCombiner::mergeConsecutiveStores(
StoreSDNode *St) {
18279 if (ElementSizeBytes * 8 != (int64_t)MemVT.
getSizeInBits())
18285 const StoreSource StoreSrc = getStoreSource(StoredVal);
18292 getStoreMergeCandidates(St, StoreNodes, RootNode);
18295 if (StoreNodes.
size() < 2)
18301 return LHS.OffsetFromBase <
RHS.OffsetFromBase;
18305 Attribute::NoImplicitFloat);
18308 cast<LoadSDNode>(StoredVal)->isNonTemporal();
18317 bool MadeChange =
false;
18318 while (StoreNodes.
size() > 1) {
18319 unsigned NumConsecutiveStores =
18320 getConsecutiveStores(StoreNodes, ElementSizeBytes);
18322 if (NumConsecutiveStores == 0)
18326 assert(NumConsecutiveStores >= 2 &&
"Expected at least 2 stores");
18327 switch (StoreSrc) {
18329 MadeChange |= tryStoreMergeOfConstants(StoreNodes, NumConsecutiveStores,
18330 MemVT, RootNode, AllowVectors);
18333 case StoreSource::Extract:
18334 MadeChange |= tryStoreMergeOfExtracts(StoreNodes, NumConsecutiveStores,
18339 MadeChange |= tryStoreMergeOfLoads(StoreNodes, NumConsecutiveStores,
18340 MemVT, RootNode, AllowVectors,
18341 IsNonTemporalStore, IsNonTemporalLoad);
18356 if (
ST->isTruncatingStore()) {
18358 ST->getBasePtr(),
ST->getMemoryVT(),
18359 ST->getMemOperand());
18361 ReplStore = DAG.
getStore(BetterChain, SL,
ST->getValue(),
ST->getBasePtr(),
18362 ST->getMemOperand());
18370 AddToWorklist(Token.
getNode());
18373 return CombineTo(
ST, Token,
false);
18410 bitcastToAPInt().getZExtValue(),
SDLoc(CFP),
18412 return DAG.
getStore(Chain,
DL, Tmp, Ptr,
ST->getMemOperand());
18424 Ptr,
ST->getMemOperand());
18427 if (
ST->isSimple() &&
18442 ST->getOriginalAlign(), MMOFlags, AAInfo);
18445 ST->getPointerInfo().getWithOffset(4),
18446 ST->getOriginalAlign(), MMOFlags, AAInfo);
18464 ST->isUnindexed()) {
18465 EVT SVT =
Value.getOperand(0).getValueType();
18472 if (((!LegalOperations &&
ST->isSimple()) ||
18475 DAG, *
ST->getMemOperand())) {
18477 ST->getMemOperand());
18482 if (
Value.isUndef() &&
ST->isUnindexed())
18488 if (*Alignment >
ST->getAlign() &&
18489 isAligned(*Alignment,
ST->getSrcValueOffset())) {
18492 ST->getMemoryVT(), *Alignment,
18493 ST->getMemOperand()->getFlags(),
ST->getAAInfo());
18503 if (
SDValue NewST = TransformFPLoadStorePair(
N))
18510 if (
ST->isUnindexed()) {
18513 if (findBetterNeighborChains(
ST)) {
18518 Chain =
ST->getChain();
18522 if (
ST->isTruncatingStore() &&
ST->isUnindexed() &&
18523 Value.getValueType().isInteger() &&
18524 (!isa<ConstantSDNode>(
Value) ||
18525 !cast<ConstantSDNode>(
Value)->isOpaque())) {
18530 Value.getOperand(0).getValueType() ==
ST->getMemoryVT() &&
18533 ST->getMemOperand());
18535 APInt TruncDemandedBits =
18537 ST->getMemoryVT().getScalarSizeInBits());
18542 AddToWorklist(
Value.getNode());
18545 ST->getMemOperand());
18565 ST->isUnindexed() &&
ST->isSimple() &&
18576 if (
StoreSDNode *ST1 = dyn_cast<StoreSDNode>(Chain)) {
18577 if (
ST->isUnindexed() &&
ST->isSimple() &&
18578 ST1->isUnindexed() && ST1->isSimple()) {
18580 ST1->getValue() ==
Value &&
ST->getMemoryVT() == ST1->getMemoryVT() &&
18581 ST->getAddressSpace() == ST1->getAddressSpace()) {
18588 !ST1->getBasePtr().isUndef() &&
18591 !
ST->getMemoryVT().isScalableVector() &&
18592 !ST1->getMemoryVT().isScalableVector() &&
18593 ST->getAddressSpace() == ST1->getAddressSpace()) {
18596 unsigned STBitSize =
ST->getMemoryVT().getFixedSizeInBits();
18597 unsigned ChainBitSize = ST1->getMemoryVT().getFixedSizeInBits();
18602 if (STBase.contains(DAG, STBitSize, ChainBase, ChainBitSize)) {
18603 CombineTo(ST1, ST1->getChain());
18616 ST->getMemoryVT(), LegalOperations)) {
18618 Ptr,
ST->getMemoryVT(),
ST->getMemOperand());
18629 bool Changed = mergeConsecutiveStores(
ST);
18630 if (!Changed)
break;
18639 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
18647 if (isa<ConstantFPSDNode>(
ST->getValue())) {
18648 if (
SDValue NewSt = replaceStoreOfFPConstant(
ST))
18655 return ReduceLoadOpStoreWidth(
N);
18659 const auto *LifetimeEnd = cast<LifetimeSDNode>(
N);
18660 if (!LifetimeEnd->hasOffset())
18664 LifetimeEnd->getOffset(),
false);
18668 while (!Chains.
empty()) {
18687 if (!
ST->isSimple() ||
ST->isIndexed())
18689 const TypeSize StoreSize =
ST->getMemoryVT().getStoreSize();
18697 if (LifetimeEndBase.contains(DAG, LifetimeEnd->getSize() * 8, StoreBase,
18700 dbgs() <<
"\nwithin LIFETIME_END of : ";
18701 LifetimeEndBase.dump();
dbgs() <<
"\n");
18702 CombineTo(
ST,
ST->getChain());
18745 if (!
ST->isSimple())
18778 !
Lo.getOperand(0).getValueType().isScalarInteger() ||
18779 Lo.getOperand(0).getValueSizeInBits() > HalfValBitSize ||
18781 !
Hi.getOperand(0).getValueType().isScalarInteger() ||
18782 Hi.getOperand(0).getValueSizeInBits() > HalfValBitSize)
18788 ?
Lo.getOperand(0).getValueType()
18789 :
Lo.getValueType();
18791 ?
Hi.getOperand(0).getValueType()
18792 :
Hi.getValueType();
18809 ST->getOriginalAlign(), MMOFlags, AAInfo);
18813 St0,
DL,
Hi, Ptr,
ST->getPointerInfo().getWithOffset(HalfValBitSize / 8),
18814 ST->getOriginalAlign(), MMOFlags, AAInfo);
18819 SDValue DAGCombiner::combineInsertEltToShuffle(
SDNode *
N,
unsigned InsIndex) {
18821 "Expected extract_vector_elt");
18822 SDValue InsertVal =
N->getOperand(1);
18831 isa<ConstantSDNode>(InsertVal.
getOperand(1))) {
18842 int ElementOffset = -1;
18851 while (!ArgWorkList.
empty()) {
18854 std::tie(ArgOffset, ArgVal) = ArgWorkList.
pop_back_val();
18856 if (ArgVal == InsertVal0) {
18857 ElementOffset = ArgOffset;
18863 int CurrentArgOffset =
18867 CurrentArgOffset -= Step;
18873 assert(CurrentArgOffset == ArgOffset);
18877 if (ElementOffset != -1) {
18880 auto *ExtrIndex = cast<ConstantSDNode>(InsertVal.
getOperand(1));
18881 NewMask[InsIndex] = ElementOffset + ExtrIndex->getZExtValue();
18882 assert(NewMask[InsIndex] <
18884 NewMask[InsIndex] >= 0 &&
"NewMask[InsIndex] is out of bound");
18890 return LegalShuffle;
18903 SDValue DestVec =
N->getOperand(0);
18909 if (NumSrcElts == 1)
18912 unsigned NumMaskVals = ExtendRatio * NumSrcElts;
18920 for (
unsigned i = 0; i != NumMaskVals; ++i) {
18921 if (i / NumSrcElts == InsIndex)
18922 Mask[i] = (i % NumSrcElts) + NumMaskVals;
18937 ConcatOps[0] = SubVec;
18943 AddToWorklist(PaddedSubV.
getNode());
18944 AddToWorklist(DestVecBC.
getNode());
18945 AddToWorklist(Shuf.
getNode());
18956 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
18989 unsigned Elt = IndexC->getZExtValue();
18990 if (
SDValue Shuf = combineInsertEltToShuffle(
N, Elt))
19001 && isa<ConstantSDNode>(InVec.
getOperand(2))) {
19003 if (Elt < OtherElt) {
19007 AddToWorklist(NewOp.
getNode());
19026 }
else if (InVec.
isUndef()) {
19031 assert(Ops.
size() == NumElts &&
"Unexpected vector size");
19034 if (Elt < Ops.
size()) {
19037 EVT OpVT = Ops[0].getValueType();
19045 SDValue DAGCombiner::scalarizeExtractedVectorLoad(
SDNode *EVE,
EVT InVecVT,
19068 if (
auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo)) {
19069 int Elt = ConstEltNo->getZExtValue();
19080 bool IsFast =
false;
19099 if (ResultVT.
bitsGT(VecEltVT)) {
19107 OriginalLoad->
getChain(), NewPtr, MPI, VecEltVT,
19110 Chain =
Load.getValue(1);
19113 VecEltVT,
SDLoc(EVE), OriginalLoad->
getChain(), NewPtr, MPI, Alignment,
19115 Chain =
Load.getValue(1);
19116 if (ResultVT.
bitsLT(VecEltVT))
19121 WorklistRemover DeadNodes(*
this);
19126 AddToWorklist(EVE);
19129 AddToWorklistWithUsers(
Load.getNode());
19137 bool LegalOperations) {
19141 auto *IndexC = dyn_cast<ConstantSDNode>(
Index);
19172 EVT ScalarVT =
N->getValueType(0);
19206 auto *IndexC = dyn_cast<ConstantSDNode>(
Index);
19218 "BUILD_VECTOR used for scalable vectors");
19219 unsigned IndexVal =
19225 if (ScalarVT == InEltVT)
19248 unsigned ExtractIndex = IndexC->getZExtValue();
19250 unsigned BCTruncElt = IsLE ? 0 : NumElts - 1;
19261 "Extract element and scalar to vector can't change element type "
19262 "from FP to integer.");
19263 unsigned XBitWidth =
X.getValueSizeInBits();
19264 BCTruncElt = IsLE ? 0 : XBitWidth / VecEltBitWidth - 1;
19269 if (ExtractIndex == BCTruncElt && XBitWidth > VecEltBitWidth) {
19270 assert(XBitWidth % VecEltBitWidth == 0 &&
19271 "Scalar bitwidth must be a multiple of vector element bitwidth");
19287 auto *Shuf = cast<ShuffleVectorSDNode>(VecOp);
19289 int OrigElt = Shuf->getMaskElt(IndexC->getZExtValue());
19297 if (OrigElt < (
int)NumElts) {
19301 OrigElt -= NumElts;
19317 if (!LegalOperations ||
19329 return Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
19330 Use->getOperand(0) == VecOp &&
19331 isa<ConstantSDNode>(Use->getOperand(1));
19335 auto *CstElt = cast<ConstantSDNode>(
Use->getOperand(1));
19336 if (CstElt->getAPIntValue().ult(NumElts))
19337 DemandedElts.
setBit(CstElt->getZExtValue());
19359 bool BCNumEltsChanged =
false;
19374 BCNumEltsChanged =
true;
19380 if (!LegalOperations && !IndexC && VecOp.
hasOneUse() &&
19383 auto *
VecLoad = dyn_cast<LoadSDNode>(VecOp);
19385 return scalarizeExtractedVectorLoad(
N, VecVT,
Index,
VecLoad);
19390 if (!LegalOperations || !IndexC)
19396 int Elt = IndexC->getZExtValue();
19399 LN0 = cast<LoadSDNode>(VecOp);
19407 LN0 = cast<LoadSDNode>(VecOp.
getOperand(0));
19409 if (
auto *Shuf = dyn_cast<ShuffleVectorSDNode>(VecOp)) {
19420 if (BCNumEltsChanged)
19424 int Idx = (Elt > (int)NumElts) ? -1 : Shuf->getMaskElt(Elt);
19435 LN0 = cast<LoadSDNode>(VecOp);
19436 Elt = (Idx < (int)NumElts) ? Idx : Idx - (int)NumElts;
19456 Index.getValueType());
19474 return scalarizeExtractedVectorLoad(
N, VecVT,
Index, LN0);
19478 SDValue DAGCombiner::reduceBuildVecExtToExtBuildVec(
SDNode *
N) {
19488 unsigned NumInScalars =
N->getNumOperands();
19490 EVT VT =
N->getValueType(0);
19498 bool AllAnyExt =
true;
19500 for (
unsigned i = 0; i != NumInScalars; ++i) {
19503 if (
In.isUndef())
continue;
19509 if (!ZeroExt && !AnyExt) {
19515 EVT InTy =
In.getOperand(0).getValueType();
19521 else if (InTy != SourceType) {
19528 AllAnyExt &= AnyExt;
19535 bool ValidTypes = SourceType !=
MVT::Other &&
19551 assert(ElemRatio > 1 &&
"Invalid element size ratio");
19559 for (
unsigned i = 0,
e =
N->getNumOperands(); i !=
e; ++i) {
19563 Cast.
isUndef()) &&
"Invalid cast opcode");
19569 unsigned Index = isLE ? (i * ElemRatio) :
19570 (i * ElemRatio + (ElemRatio - 1));
19579 "Invalid vector size");
19607 EVT VT =
N->getValueType(0);
19615 unsigned NumInScalars =
N->getNumOperands();
19618 auto PeekThroughBitcast = [](
SDValue Op) {
19620 return Op.getOperand(0);
19626 for (
unsigned i = 0; i != NumInScalars; ++i) {
19627 SDValue In = PeekThroughBitcast(
N->getOperand(i));
19629 if (
In.isUndef())
continue;
19634 In = PeekThroughBitcast(
In.getOperand(0));
19645 SDValue part = PeekThroughBitcast(
In.getOperand(0));
19649 }
else if (Src != part) {
19654 SDValue ShiftAmtVal =
In.getOperand(1);
19655 if (!isa<ConstantSDNode>(ShiftAmtVal))
19658 uint64_t ShiftAmt =
In.getNode()->getConstantOperandVal(1);
19661 if (ShiftAmt != i * ScalarTypeBitsize)
19667 if (Src.getValueType().getSizeInBits() != VT.
getSizeInBits())
19676 unsigned LeftIdx,
bool DidSplitVec) {
19679 EVT VT =
N->getValueType(0);
19684 unsigned ShuffleNumElems = NumElems;
19694 assert(InVT2Size <= InVT1Size &&
19695 "Inputs must be sorted to be in non-increasing vector size order.");
19699 if (InVT1 != VT || InVT2 != VT) {
19700 if ((VTSize % InVT1Size == 0) && InVT1 == InVT2) {
19703 unsigned NumConcats = VTSize / InVT1Size;
19704 assert(NumConcats >= 2 &&
"Concat needs at least two inputs!");
19706 ConcatOps[0] = VecIn1;
19707 ConcatOps[1] = VecIn2 ? VecIn2 : DAG.
getUNDEF(InVT1);
19710 }
else if (InVT1Size == VTSize * 2) {
19722 Vec2Offset = NumElems;
19724 assert(InVT2Size <= InVT1Size &&
19725 "Second input is not going to be larger than the first one.");
19731 if (LegalOperations &&
19738 if (InVT1 != InVT2) {
19742 DAG.
getUNDEF(InVT1), VecIn2, ZeroIdx);
19744 ShuffleNumElems = NumElems * 2;
19746 }
else if (InVT2Size * 2 == VTSize && InVT1Size == VTSize) {
19748 ConcatOps[0] = VecIn2;
19765 for (
unsigned i = 0; i != NumElems; ++i) {
19766 if (VectorMask[i] <= 0)
19769 unsigned ExtIndex =
N->getOperand(i).getConstantOperandVal(1);
19770 if (VectorMask[i] == (
int)LeftIdx) {
19771 Mask[i] = ExtIndex;
19772 }
else if (VectorMask[i] == (
int)LeftIdx + 1) {
19773 Mask[i] = Vec2Offset + ExtIndex;
19786 if (ShuffleNumElems > NumElems)
19799 for (
int i = 0; i != NumBVOps; ++i) {
19831 if (DestSize % SrcSize != 0 ||
19837 int ZextRatio = DestSize / SrcSize;
19838 int NumMaskElts = NumBVOps * ZextRatio;
19840 for (
int i = 0; i != NumMaskElts; ++i) {
19841 if (i / ZextRatio == ZextElt) {
19846 if (i % ZextRatio == 0)
19849 ShufMask[i] = NumMaskElts;
19863 ZeroVec, ShufMask, DAG);
19870 template <
typename R,
typename T>
19872 auto I =
find(Range, Val);
19873 if (
I == Range.end())
19874 return static_cast<decltype(std::distance(Range.
begin(),
I))
>(-1);
19875 return std::distance(Range.begin(),
I);
19883 EVT VT =
N->getValueType(0);
19896 bool UsesZeroVector =
false;
19897 unsigned NumElems =
N->getNumOperands();
19909 for (
unsigned i = 0; i != NumElems; ++i) {
19919 UsesZeroVector =
true;
19927 !isa<ConstantSDNode>(
Op.getOperand(1)))
19929 SDValue ExtractedFromVec =
Op.getOperand(0);
19934 const APInt &ExtractIdx =
Op.getConstantOperandAPInt(1);
19948 Idx = VecIn.
size();
19952 VectorMask[i] = Idx;
19956 if (VecIn.
size() < 2)
19963 bool DidSplitVec =
false;
19964 if (VecIn.
size() == 2) {
19965 unsigned MaxIndex = 0;
19966 unsigned NearestPow2 = 0;
19971 for (
unsigned i = 0; i < NumElems; i++) {
19972 if (VectorMask[i] <= 0)
19974 unsigned Index =
N->getOperand(i).getConstantOperandVal(1);
19975 IndexVec[i] =
Index;
19980 if (InVT.
isSimple() && NearestPow2 > 2 && MaxIndex < NearestPow2 &&
19981 NumElems * 2 < NearestPow2) {
19982 unsigned SplitSize = NearestPow2 / 2;
19995 DidSplitVec =
true;
19997 for (
unsigned i = 0; i < NumElems; i++) {
19998 if (VectorMask[i] <= 0)
20000 VectorMask[i] = (IndexVec[i] < SplitSize) ? 1 : 2;
20018 for (
int &SourceVectorIndex : VectorMask) {
20019 if (SourceVectorIndex <= 0)
20021 unsigned Idx =
getFirstIndexOf(SortedVecIn, VecIn[SourceVectorIndex]);
20022 assert(Idx > 0 && Idx < SortedVecIn.size() &&
20023 VecIn[SourceVectorIndex] == SortedVecIn[Idx] &&
"Remapping failure");
20024 SourceVectorIndex = Idx;
20045 for (
unsigned In = 0, Len = (VecIn.
size() / 2);
In < Len; ++
In) {
20046 unsigned LeftIdx = 2 *
In + 1;
20047 SDValue VecLeft = VecIn[LeftIdx];
20049 (LeftIdx + 1) < VecIn.
size() ? VecIn[LeftIdx + 1] :
SDValue();
20051 if (
SDValue Shuffle = createBuildVecShuffle(
DL,
N, VectorMask, VecLeft,
20052 VecRight, LeftIdx, DidSplitVec))
20060 if (UsesZeroVector)
20065 if (Shuffles.
size() == 1)
20066 return Shuffles[0];
20069 for (
int &Vec : VectorMask)
20071 Vec = Shuffles.
size() - 1;
20073 Vec = (Vec - 1) / 2;
20087 if (Shuffles.
size() % 2)
20090 for (
unsigned CurSize = Shuffles.
size(); CurSize > 1; CurSize /= 2) {
20092 Shuffles[CurSize] = DAG.
getUNDEF(VT);
20095 for (
unsigned In = 0, Len = CurSize / 2;
In < Len; ++
In) {
20099 for (
unsigned i = 0; i != NumElems; ++i) {
20100 if (VectorMask[i] == Left) {
20102 VectorMask[i] =
In;
20103 }
else if (VectorMask[i] == Right) {
20104 Mask[i] = i + NumElems;
20105 VectorMask[i] =
In;
20113 return Shuffles[0];
20121 if (LegalOperations)
20124 EVT VT =
N->getValueType(0);
20126 bool FoundZeroExtend =
false;
20128 auto checkElem = [&](
SDValue Op) -> int64_t {
20129 unsigned Opc =
Op.getOpcode();
20134 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op.getOperand(0).getOperand(1)))
20135 return C->getZExtValue();
20143 int64_t
Offset = checkElem(Op0);
20147 unsigned NumElems =
N->getNumOperands();
20149 EVT InSVT =
In.getValueType().getScalarType();
20157 for (
unsigned i = 1; i != NumElems; ++i) {
20158 if ((
Offset + i) != checkElem(
N->getOperand(i)))
20170 EVT VT =
N->getValueType(0);
20184 if (!LegalOperations) {
20187 EVT SrcVT = Splat.getValueType();
20203 if (!LegalTypes && (
N->getNumOperands() > 1)) {
20208 if (
auto CNode = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
20209 return CNode->getZExtValue();
20213 int Offset = checkElem(Op0);
20214 for (
unsigned i = 0; i <
N->getNumOperands(); ++i) {
20215 if (
Offset + i != checkElem(
N->getOperand(i))) {
20225 ((
Offset %
N->getValueType(0).getVectorNumElements()) ==
20231 if (
SDValue V = convertBuildVecZextToZext(
N))
20234 if (
SDValue V = reduceBuildVecExtToExtBuildVec(
N))
20237 if (
SDValue V = reduceBuildVecTruncToBitCast(
N))
20240 if (
SDValue V = reduceBuildVecToShuffle(
N))
20247 assert(!V.isUndef() &&
"Splat of undef should have been handled earlier");
20256 EVT OpVT =
N->getOperand(0).getValueType();
20263 EVT VT =
N->getValueType(0);
20270 bool AnyInteger =
false;
20271 bool AnyFP =
false;
20274 !
Op.getOperand(0).getValueType().isVector())
20283 EVT LastOpVT = Ops.
back().getValueType();
20300 if (
Op.getValueType() == SVT)
20320 EVT VT =
N->getValueType(0);
20330 if (!FirstConcat) {
20331 SubVT =
Op.getOperand(0).getValueType();
20337 if (SubVT !=
Op.getOperand(0).getValueType())
20340 assert(FirstConcat &&
"Concat of all-undefs found");
20344 if (
Op.isUndef()) {
20348 ConcatOps.
append(
Op->op_begin(),
Op->op_end());
20358 EVT VT =
N->getValueType(0);
20359 EVT OpVT =
N->getOperand(0).getValueType();
20375 if (
Op.isUndef()) {
20376 Mask.append((
unsigned)NumOpElts, -1);
20385 int ExtIdx =
Op.getConstantOperandVal(1);
20394 Mask.append((
unsigned)NumOpElts, -1);
20405 if (0 == (NumExtElts % NumElts))
20406 ExtIdx /= (NumExtElts / NumElts);
20407 else if (0 == (NumElts % NumExtElts))
20408 ExtIdx *= (NumElts / NumExtElts);
20413 if (SV0.
isUndef() || SV0 == ExtVec) {
20415 for (
int i = 0; i != NumOpElts; ++i)
20416 Mask.push_back(i + ExtIdx);
20417 }
else if (SV1.
isUndef() || SV1 == ExtVec) {
20419 for (
int i = 0; i != NumOpElts; ++i)
20420 Mask.push_back(i + ExtIdx + NumElts);
20432 unsigned CastOpcode =
N->getOperand(0).getOpcode();
20433 switch (CastOpcode) {
20449 EVT SrcVT =
N->getOperand(0).getOperand(0).getValueType();
20457 if (
Op.getOpcode() != CastOpcode || !
Op.hasOneUse() ||
20458 Op.getOperand(0).getValueType() != SrcVT)
20466 EVT VT =
N->getValueType(0);
20471 switch (CastOpcode) {
20491 return DAG.
getNode(CastOpcode,
DL, VT, NewConcat);
20496 if (
N->getNumOperands() == 1)
20497 return N->getOperand(0);
20500 EVT VT =
N->getValueType(0);
20506 [](
const SDValue &
Op) { return Op.isUndef(); })) {
20508 assert(
In.getValueType().isVector() &&
"Must concat vectors");
20513 unsigned NumOps =
N->getNumOperands() *
In.getNumOperands();
20525 EVT SVT =
Scalar.getValueType().getVectorElementType();
20526 if (SVT ==
Scalar.getOperand(0).getValueType())
20531 if (!
Scalar.getValueType().isVector()) {
20550 if (VNTNumElms < 2)
20566 auto IsBuildVectorOrUndef = [](
const SDValue &
Op) {
20577 bool FoundMinVT =
false;
20580 EVT OpSVT =
Op.getOperand(0).getValueType();
20581 MinVT = (!FoundMinVT || OpSVT.
bitsLE(MinVT)) ? OpSVT : MinVT;
20584 assert(FoundMinVT &&
"Concat vector type mismatch");
20588 EVT OpVT =
Op.getValueType();
20597 Opnds.
append(
Op->op_begin(),
Op->op_begin() + NumElts);
20599 for (
unsigned i = 0; i != NumElts; ++i)
20607 "Concat vector type mismatch");
20636 unsigned PartNumElem =
20637 N->getOperand(0).getValueType().getVectorMinNumElements();
20639 for (
unsigned i = 0,
e =
N->getNumOperands(); i !=
e; ++i) {
20650 if (SingleSource.
getNode()) {
20651 if (
Op.getOperand(0) != SingleSource)
20654 SingleSource =
Op.getOperand(0);
20664 unsigned IdentityIndex = i * PartNumElem;
20665 if (
Op.getConstantOperandAPInt(1) != IdentityIndex)
20670 return SingleSource;
20682 auto *IndexC = dyn_cast<ConstantSDNode>(
Index);
20694 bool LegalOperations) {
20697 unsigned BinOpcode = BinOp.
getOpcode();
20703 if (VecVT != Bop0.
getValueType() || VecVT != Bop1.getValueType())
20717 if (!Sub0 || !Sub1)
20723 return DAG.
getNode(BinOpcode,
SDLoc(Extract), SubVT, Sub0, Sub1,
20730 bool LegalOperations) {
20738 auto *ExtractIndexC = dyn_cast<ConstantSDNode>(Extract->
getOperand(1));
20739 if (!ExtractIndexC)
20755 if (
C &&
C->getValueAPF().isNegZero())
20768 unsigned ExtractIndex = ExtractIndexC->getZExtValue();
20770 "Extract index is not a multiple of the vector length.");
20775 if (WideWidth % NarrowWidth != 0)
20780 unsigned NarrowingRatio = WideWidth / NarrowWidth;
20782 if (WideNumElts % NarrowingRatio != 0)
20787 WideNumElts / NarrowingRatio);
20813 if (NarrowingRatio != 2)
20828 return V.getOperand(ConcatOpNum);
20834 if (SubVecL || SubVecR) {
20865 auto *Ld = dyn_cast<LoadSDNode>(Extract->
getOperand(0));
20881 assert(
Index % NumElts == 0 &&
"The extract subvector index is not a "
20882 "multiple of the result's element count");
20901 if (
Offset.isScalable()) {
20924 bool LegalOperations) {
20926 "Must only be called on EXTRACT_SUBVECTOR's");
20931 EVT NarrowVT =
N->getValueType(0);
20937 auto *WideShuffleVector = dyn_cast<ShuffleVectorSDNode>(N0);
20938 if (!WideShuffleVector)
20942 if (!WideShuffleVector->hasOneUse())
20946 if (LegalOperations &&
20950 uint64_t FirstExtractedEltIdx =
N->getConstantOperandVal(1);
20952 assert((FirstExtractedEltIdx % NumEltsExtracted) == 0 &&
20953 "Extract index is not a multiple of the output vector length.");
20958 NewMask.
reserve(NumEltsExtracted);
20960 DemandedSubvectors;
20963 for (
int M : WideShuffleVector->getMask().slice(FirstExtractedEltIdx,
20964 NumEltsExtracted)) {
20965 assert((M >= -1) && (M < (2 * WideNumElts)) &&
20966 "Out-of-bounds shuffle mask?");
20975 int WideShufOpIdx = M / WideNumElts;
20977 int OpEltIdx = M % WideNumElts;
20979 assert((OpEltIdx + WideShufOpIdx * WideNumElts) == M &&
20980 "Shuffle mask vector decomposition failure.");
20983 int OpSubvecIdx = OpEltIdx / NumEltsExtracted;
20985 int OpEltIdxInSubvec = OpEltIdx % NumEltsExtracted;
20987 assert((OpEltIdxInSubvec + OpSubvecIdx * NumEltsExtracted) == OpEltIdx &&
20988 "Shuffle mask subvector decomposition failure.");
20990 assert((OpEltIdxInSubvec + OpSubvecIdx * NumEltsExtracted +
20991 WideShufOpIdx * WideNumElts) == M &&
20992 "Shuffle mask full decomposition failure.");
20994 SDValue Op = WideShuffleVector->getOperand(WideShufOpIdx);
20996 if (
Op.isUndef()) {
21003 if (OpSubvecIdx != 0)
21006 const std::pair<SDValue, int> DemandedSubvector =
21007 std::make_pair(
Op, OpSubvecIdx);
21009 if (DemandedSubvectors.insert(DemandedSubvector)) {
21010 if (DemandedSubvectors.size() > 2)
21013 int Index = NumEltsExtracted * OpSubvecIdx;
21021 getFirstIndexOf(DemandedSubvectors.getArrayRef(), DemandedSubvector);
21022 assert((NewOpIdx == 0 || NewOpIdx == 1) &&
"Unexpected operand index.");
21024 int AdjM = OpEltIdxInSubvec + NewOpIdx * NumEltsExtracted;
21027 assert(NewMask.
size() == (
unsigned)NumEltsExtracted &&
"Produced bad mask.");
21028 assert(DemandedSubvectors.size() <= 2 &&
21029 "Should have ended up demanding at most two subvectors.");
21032 if (DemandedSubvectors.empty())
21046 for (
const std::pair<SDValue /*Op*/, int /*SubvectorIndex*/>
21047 &DemandedSubvector : DemandedSubvectors) {
21049 int Index = NumEltsExtracted * DemandedSubvector.second;
21052 DemandedSubvector.first, IndexC));
21055 "Should end up with either one or two ops");
21058 if (NewOps.
size() == 1)
21065 EVT NVT =
N->getValueType(0);
21067 uint64_t ExtIdx =
N->getConstantOperandVal(1);
21097 if ((SrcNumElts % DestNumElts) == 0) {
21098 unsigned SrcDestRatio = SrcNumElts / DestNumElts;
21110 if ((DestNumElts % SrcNumElts) == 0) {
21111 unsigned DestSrcRatio = DestNumElts / SrcNumElts;
21116 if ((ExtIdx % DestSrcRatio) == 0) {
21118 unsigned IndexValScaled = ExtIdx / DestSrcRatio;
21145 "Concat and extract subvector do not change element type");
21146 assert((ExtIdx % ExtNumElts) == 0 &&
21147 "Extract index is not a multiple of the input vector length.");
21150 unsigned ConcatOpIdx = ExtIdx / ConcatSrcNumElts;
21164 ConcatSrcNumElts % ExtNumElts == 0) {
21166 unsigned NewExtIdx = ExtIdx - ConcatOpIdx * ConcatSrcNumElts;
21167 assert(NewExtIdx + ExtNumElts <= ConcatSrcNumElts &&
21168 "Trying to extract from >1 concat operand?");
21169 assert(NewExtIdx % ExtNumElts == 0 &&
21170 "Extract index is not a multiple of the input vector length.");
21189 if (ExtractSize % EltSize == 0) {
21190 unsigned NumElems = ExtractSize / EltSize;
21193 NumElems == 1 ? EltVT
21201 if (NumElems == 1) {
21203 if (EltVT != Src.getValueType())
21210 V->
ops().slice(IdxVal, NumElems));
21220 if (!NVT.
bitsEq(SmallVT))
21268 unsigned HalfNumElts = NumElts / 2;
21271 for (
unsigned i = 0; i != NumElts; ++i) {
21275 if ((
Mask[i] % NumElts) >= HalfNumElts)
21277 int M =
Mask[i] < (int)NumElts ?
Mask[i] :
Mask[i] - (
int)HalfNumElts;
21278 if (i < HalfNumElts)
21281 Mask1[i - HalfNumElts] = M;
21304 EVT VT =
N->getValueType(0);
21315 unsigned NumConcats = NumElts / NumElemsPerConcat;
21317 auto IsUndefMaskElt = [](
int i) {
return i == -1; };
21322 if (NumElemsPerConcat * 2 == NumElts && N1.
isUndef() &&
21327 Mask.slice(0, NumElemsPerConcat));
21334 for (
unsigned I = 0;
I != NumConcats; ++
I) {
21335 unsigned Begin =
I * NumElemsPerConcat;
21345 for (
int i = 0; i != (int)NumElemsPerConcat; ++i) {
21346 if (IsUndefMaskElt(SubMask[i]))
21348 if ((SubMask[i] % (
int)NumElemsPerConcat) != i)
21350 int EltOpIdx = SubMask[i] / NumElemsPerConcat;
21351 if (0 <= OpIdx && EltOpIdx != OpIdx)
21355 assert(0 <= OpIdx &&
"Unknown concat_vectors op");
21409 bool IsSplat =
false;
21410 auto *BV0 = dyn_cast<BuildVectorSDNode>(N0);
21411 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
21413 if (
SDValue Splat0 = BV0->getSplatValue())
21414 IsSplat = (Splat0 == BV1->getSplatValue());
21418 for (
int M : SVN->
getMask()) {
21421 int Idx = M < (int)NumElts ? M : M - NumElts;
21422 SDValue &S = (M < (int)NumElts ? N0 : N1);
21424 Op = S.getOperand(Idx);
21439 if (!IsSplat && !DuplicateOps.
insert(
Op).second)
21450 SVT = (SVT.
bitsLT(
Op.getValueType()) ?
Op.getValueType() : SVT);
21466 bool LegalOperations) {
21480 auto isAnyExtend = [&
Mask, &NumElts](
unsigned Scale) {
21481 for (
unsigned i = 0; i != NumElts; ++i) {
21484 if ((i % Scale) == 0 &&
Mask[i] == (int)(i / Scale))
21493 for (
unsigned Scale = 2; Scale < NumElts; Scale *= 2) {
21495 if (NumElts % Scale != 0)
21497 if (!isAnyExtend(Scale))
21505 if (!LegalOperations ||
21509 SDLoc(SVN), OutVT, N0));
21544 if (ExtDstSizeInBits % ExtSrcSizeInBits != 0)
21546 unsigned ExtScale = ExtDstSizeInBits / ExtSrcSizeInBits;
21551 auto isTruncate = [&
Mask, &NumElts](
unsigned Scale) {
21552 for (
unsigned i = 0; i != NumElts; ++i) {
21555 if ((i * Scale) < NumElts &&
Mask[i] == (int)(i * Scale))
21565 if (EltSizeInBits != ExtSrcSizeInBits)
21570 if (isTruncate(ExtScale))
21585 auto *Splat = dyn_cast<ShuffleVectorSDNode>(Shuf->
getOperand(0));
21586 if (!Splat || !Splat->isSplat())
21591 assert(ShufMask.
size() == SplatMask.
size() &&
"Mask length mismatch");
21609 auto CanSimplifyToExistingSplat = [](
ArrayRef<int> UserMask,
21611 for (
unsigned i = 0,
e = UserMask.
size(); i !=
e; ++i)
21612 if (UserMask[i] != -1 && SplatMask[i] == -1 &&
21613 SplatMask[UserMask[i]] != -1)
21617 if (CanSimplifyToExistingSplat(ShufMask, SplatMask))
21623 for (
int Idx : ShufMask)
21624 NewMask.
push_back(Idx == -1 ? -1 : SplatMask[Idx]);
21627 Splat->getOperand(0), Splat->getOperand(1),
21637 auto *InnerShuf = dyn_cast<ShuffleVectorSDNode>(OuterShuf->
getOperand(0));
21638 if (!InnerShuf || !InnerShuf->getOperand(1).isUndef())
21643 unsigned NumElts = OuterMask.
size();
21644 assert(NumElts == InnerMask.
size() &&
"Mask length mismatch");
21646 int SplatIndex = -1;
21647 for (
unsigned i = 0; i != NumElts; ++i) {
21649 int OuterMaskElt = OuterMask[i];
21650 if (OuterMaskElt == -1)
21654 int InnerMaskElt = InnerMask[OuterMaskElt];
21655 if (InnerMaskElt == -1)
21659 if (SplatIndex == -1)
21660 SplatIndex = InnerMaskElt;
21663 if (SplatIndex != InnerMaskElt)
21666 CombinedMask[i] = InnerMaskElt;
21668 assert((
all_of(CombinedMask, [](
int M) {
return M == -1; }) ||
21670 "Expected a splat mask");
21674 assert(VT == InnerShuf->getValueType(0) &&
"Expected matching shuffle types");
21679 InnerShuf->getOperand(1), CombinedMask);
21687 int MaskSize =
Mask.size();
21688 int EltFromOp0 = -1;
21693 for (
int i = 0; i != MaskSize; ++i) {
21694 if (
Mask[i] >= 0 &&
Mask[i] < MaskSize) {
21696 if (EltFromOp0 != -1)
21699 }
else if (
Mask[i] != i + MaskSize) {
21719 if (ShufOp0Index == -1) {
21723 if (ShufOp0Index == -1)
21727 Mask = CommutedMask;
21736 "Shuffle mask value must be from operand 0");
21740 auto *InsIndexC = dyn_cast<ConstantSDNode>(Op0.
getOperand(2));
21741 if (!InsIndexC || InsIndexC->getSExtValue() !=
Mask[ShufOp0Index])
21765 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(Shuf->
getOperand(0));
21771 for (
int i = 0,
e = (
int)
Mask.size(); i !=
e; ++i) {
21775 assert(
Mask[i] >= 0 &&
Mask[i] <
e &&
"Unexpected shuffle mask value");
21779 if (Mask0[
Mask[i]] != Mask0[i])
21788 EVT VT =
N->getValueType(0);
21813 bool Changed =
false;
21815 for (
unsigned i = 0; i != NumElts; ++i) {
21817 if (Idx >= (
int)NumElts) {
21871 "BUILD_VECTOR has wrong number of operands");
21873 bool AllSame =
true;
21874 for (
unsigned i = 0; i != NumElts; ++i) {
21881 if (!
Base.getNode())
21883 for (
unsigned i = 0; i != NumElts; ++i) {
21938 int HalfNumElts = (int)NumElts / 2;
21940 for (
unsigned i = 0; i != NumElts; ++i) {
21942 if (Idx >= HalfNumElts) {
21943 assert(Idx < (
int)NumElts &&
"Shuffle mask chooses undef op");
21944 Idx -= HalfNumElts;
21965 EVT SubVT =
RHS.getOperand(0).getValueType();
21966 int NumSubVecs =
RHS.getNumOperands();
21968 assert((NumElts % NumSubElts) == 0 &&
"Subvector mismatch");
21973 if (
all_of(
Mask, [NumElts](
int M) {
return M < (int)NumElts; }))
21979 for (
int SubVec = 0; SubVec != NumSubVecs; ++SubVec) {
21980 for (
int SubIdx = 0; SubIdx != (int)NumElts; SubIdx += NumSubElts) {
21982 std::iota(InsertionMask.begin(), InsertionMask.end(), 0);
21985 std::iota(InsertionMask.begin() + SubIdx,
21986 InsertionMask.begin() + SubIdx + NumSubElts,
21987 NumElts + (SubVec * NumSubElts));
21990 bool MatchingShuffle =
true;
21991 for (
int i = 0; i != (int)NumElts; ++i) {
21992 int ExpectIdx = InsertionMask[i];
21993 int ActualIdx =
Mask[i];
21994 if (0 <= ActualIdx && ExpectIdx != ActualIdx) {
21995 MatchingShuffle =
false;
22000 if (MatchingShuffle)
22002 RHS.getOperand(SubVec),
22010 if (
SDValue InsertN1 = ShuffleToInsert(N0, N1,
Mask))
22015 if (
SDValue InsertN0 = ShuffleToInsert(N1, N0, CommuteMask))
22040 EVT ScaleVT = SVT.
bitsLT(InnerSVT) ? VT : InnerVT;
22058 for (
int M : OuterMask)
22059 NewMask.
push_back(M < 0 ? -1 : InnerMask[M]);
22085 auto MergeInnerShuffle =
22092 if (OtherSVN->isSplat())
22098 for (
unsigned i = 0; i != NumElts; ++i) {
22099 int Idx = SVN->getMaskElt(i);
22102 Mask.push_back(Idx);
22107 Idx = (Idx < (int)NumElts) ? (Idx + NumElts) : (Idx - NumElts);
22110 if (Idx < (
int)NumElts) {
22113 Idx = OtherSVN->getMaskElt(Idx);
22116 Mask.push_back(Idx);
22119 CurrentVec = (Idx < (int)NumElts) ? OtherSVN->
getOperand(0)
22128 Mask.push_back(-1);
22134 Idx = Idx % NumElts;
22135 if (!SV0.getNode() || SV0 == CurrentVec) {
22139 Mask.push_back(Idx);
22142 if (!SV1.getNode() || SV1 == CurrentVec) {
22146 Mask.push_back(Idx + NumElts);
22152 if (
auto *CurrentSVN = dyn_cast<ShuffleVectorSDNode>(CurrentVec)) {
22153 int InnerIdx = CurrentSVN->getMaskElt(Idx);
22154 if (InnerIdx < 0) {
22155 Mask.push_back(-1);
22158 SDValue InnerVec = (InnerIdx < (int)NumElts)
22162 Mask.push_back(-1);
22165 InnerIdx %= NumElts;
22166 if (InnerVec == SV0) {
22167 Mask.push_back(InnerIdx);
22170 if (InnerVec == SV1) {
22171 Mask.push_back(InnerIdx + NumElts);
22190 if (TLI.isShuffleMaskLegal(
Mask, VT))
22195 return TLI.isShuffleMaskLegal(
Mask, VT);
22207 assert(N1->getOperand(0).getValueType() == VT &&
22208 "Shuffle types don't match");
22212 bool HasSameOp0 = N0 == SV0;
22213 bool IsSV1Undef = SV1.
isUndef();
22214 if (HasSameOp0 || IsSV1Undef || N0 == SV1)
22223 cast<ShuffleVectorSDNode>(N0)->isSplat() &&
22224 !cast<ShuffleVectorSDNode>(N1)->isSplat()) {
22235 for (
int i = 0; i != 2; ++i) {
22237 N->isOnlyUserOf(
N->getOperand(i).getNode())) {
22240 auto *OtherSV = cast<ShuffleVectorSDNode>(
N->getOperand(i));
22241 assert(OtherSV->getOperand(0).getValueType() == VT &&
22242 "Shuffle types don't match");
22246 if (MergeInnerShuffle(i != 0, SVN, OtherSV,
N->getOperand(1 - i), TLI,
22264 if (TLI.isBinOp(SrcOpcode) &&
N->isOnlyUserOf(N0.
getNode()) &&
22266 (SrcOpcode == N1.getOpcode() &&
N->isOnlyUserOf(N1.getNode())))) {
22284 SDValue InnerN = Commute ? N1 : N0;
22285 SDValue Op0 = LeftOp ? Op00 : Op01;
22286 SDValue Op1 = LeftOp ? Op10 : Op11;
22291 auto *SVN0 = dyn_cast<ShuffleVectorSDNode>(Op0);
22293 MergeInnerShuffle(Commute, SVN, SVN0, Op1, TLI, SV0, SV1,
22295 (
llvm::any_of(SVN0->getMask(), [](
int M) { return M < 0; }) ||
22301 bool MergedLeft =
false;
22304 if (CanMergeInnerShuffle(LeftSV0, LeftSV1, LeftMask,
true,
false) ||
22305 CanMergeInnerShuffle(LeftSV0, LeftSV1, LeftMask,
true,
true)) {
22308 LeftMask.
assign(SVN->getMask().begin(), SVN->getMask().end());
22309 LeftSV0 = Op00, LeftSV1 = Op10;
22312 bool MergedRight =
false;
22315 if (CanMergeInnerShuffle(RightSV0, RightSV1, RightMask,
false,
false) ||
22316 CanMergeInnerShuffle(RightSV0, RightSV1, RightMask,
false,
true)) {
22317 MergedRight =
true;
22319 RightMask.
assign(SVN->getMask().begin(), SVN->getMask().end());
22320 RightSV0 = Op01, RightSV1 = Op11;
22323 if (MergedLeft || MergedRight) {
22326 VT,
DL, LeftSV0 ? LeftSV0 : DAG.
getUNDEF(VT),
22327 LeftSV1 ? LeftSV1 : DAG.
getUNDEF(VT), LeftMask);
22329 VT,
DL, RightSV0 ? RightSV0 : DAG.
getUNDEF(VT),
22330 RightSV1 ? RightSV1 : DAG.
getUNDEF(VT), RightMask);
22345 EVT VT =
N->getValueType(0);
22371 TLI.buildLegalVectorShuffle(InVecT,
SDLoc(
N), InVec,
22372 DAG.
getUNDEF(InVecT), NewMask, DAG);
22373 if (LegalShuffle) {
22377 return LegalShuffle;
22382 InVecT.getVectorElementType(),
22385 LegalShuffle, ZeroIdx);
22396 EVT VT =
N->getValueType(0);
22400 uint64_t InsIdx =
N->getConstantOperandVal(2);
22470 if ((N0.
isUndef() || N0SrcSVT == N1SrcSVT) &&
22482 }
else if ((N1SrcSVT.
getSizeInBits() % EltSizeInBits) == 0) {
22505 if (InsIdx < OtherIdx) {
22509 AddToWorklist(NewOp.
getNode());
22523 Ops[InsIdx / Factor] = N1;
22562 unsigned Opcode =
N->getOpcode();
22580 if (!TLI.isOperationLegalOrCustom(Opcode, VT) &&
22581 TLI.isOperationLegalOrCustom(NewOpcode, VT) &&
22583 return DAG.
getNode(NewOpcode,
SDLoc(
N),
N->getValueType(0), N0);
22593 bool AreAllEltsDisabled =
false;
22597 AreAllEltsDisabled |=
22601 if (!AreAllEltsDisabled)
22606 return DAG.
getUNDEF(
N->getValueType(0));
22610 if (
const auto *MemSD = dyn_cast<MemSDNode>(
N)) {
22611 if (MemSD->writeMem())
22612 return MemSD->getChain();
22613 return CombineTo(
N, DAG.
getUNDEF(
N->getValueType(0)), MemSD->getChain());
22618 return N->getOperand(0);
22630 EVT VT =
N->getValueType(0);
22637 if (LegalOperations)
22643 EVT RVT =
RHS.getValueType();
22644 unsigned NumElts =
RHS.getNumOperands();
22649 auto BuildClearMask = [&](
int Split) {
22650 int NumSubElts = NumElts * Split;
22654 for (
int i = 0; i != NumSubElts; ++i) {
22655 int EltIdx = i / Split;
22656 int SubIdx = i % Split;
22666 if (isa<ConstantSDNode>(Elt))
22667 Bits = cast<ConstantSDNode>(Elt)->getAPIntValue();
22668 else if (isa<ConstantFPSDNode>(Elt))
22669 Bits = cast<ConstantFPSDNode>(Elt)->getValueAPF().bitcastToAPInt();
22675 Bits =
Bits.extractBits(NumSubBits, (Split - SubIdx - 1) * NumSubBits);
22677 Bits =
Bits.extractBits(NumSubBits, SubIdx * NumSubBits);
22679 if (
Bits.isAllOnes())
22681 else if (
Bits == 0)
22690 if (!TLI.isVectorClearMaskLegal(Indices, ClearVT))
22704 for (
int Split = 1; Split <= MaxSplit; ++Split)
22706 if (
SDValue S = BuildClearMask(Split))
22718 unsigned Opcode =
N->getOpcode();
22719 EVT VT =
N->getValueType(0);
22726 int Index0, Index1;
22729 if (!Src0 || !Src1 || Index0 != Index1 ||
22749 Ops[Index0] = ScalarBO;
22760 EVT VT =
N->getValueType(0);
22761 assert(VT.
isVector() &&
"SimplifyVBinOp only works on vectors!");
22765 unsigned Opcode =
N->getOpcode();
22778 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(
LHS);
22779 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(
RHS);
22780 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) &&
22781 LHS.getOperand(1).isUndef() &&
RHS.getOperand(1).isUndef() &&
22784 RHS.getOperand(0), Flags);
22796 Shuf0->hasOneUse() && Shuf0->getOperand(1).isUndef() &&
22805 Shuf1->hasOneUse() && Shuf1->getOperand(1).isUndef() &&
22821 LHS.getOperand(2) ==
RHS.getOperand(2) &&
22826 EVT NarrowVT =
X.getValueType();
22827 if (NarrowVT ==
Y.getValueType() &&
22828 TLI.isOperationLegalOrCustomOrPromote(Opcode, NarrowVT,
22829 LegalOperations)) {
22842 return Op.isUndef() ||
22843 ISD::isBuildVectorOfConstantSDNodes(Op.getNode());
22852 if (ConcatWithConstantOrUndef(
LHS) && ConcatWithConstantOrUndef(
RHS) &&
22854 EVT NarrowVT =
LHS.getOperand(0).getValueType();
22855 if (NarrowVT ==
RHS.getOperand(0).getValueType() &&
22856 TLI.isOperationLegalOrCustomOrPromote(Opcode, NarrowVT)) {
22857 unsigned NumOperands =
LHS.getNumOperands();
22859 for (
unsigned i = 0; i != NumOperands; ++i) {
22862 RHS.getOperand(i)));
22880 cast<CondCodeSDNode>(N0.
getOperand(2))->get());
22885 if (
SCC.getNode()) {
22892 SCC.getOperand(0),
SCC.getOperand(1),
22893 SCC.getOperand(4), Flags);
22894 AddToWorklist(
SETCC.getNode());
22896 SCC.getOperand(2),
SCC.getOperand(3));
22924 CC = cast<CondCodeSDNode>(TheSelect->
getOperand(4))->get();
22931 CC = cast<CondCodeSDNode>(
Cmp.getOperand(2))->get();
22932 CmpLHS =
Cmp.getOperand(0);
22936 if (Zero && Zero->
isZero() &&
22940 CombineTo(TheSelect, Sqrt);
22950 if (
LHS.getOpcode() !=
RHS.getOpcode() ||
22963 if (
LHS.getOperand(0) !=
RHS.getOperand(0) ||
22983 LLD->getPointerInfo().getAddrSpace() != 0 ||
22989 !TLI.isOperationLegalOrCustom(TheSelect->
getOpcode(),
22990 LLD->getBasePtr().getValueType()))
23007 Visited.
insert(TheSelect);
23025 if ((
LLD->hasAnyUseOfValue(1) &&
23032 LLD->getBasePtr().getValueType(),
23047 if ((
LLD->hasAnyUseOfValue(1) &&
23054 LLD->getBasePtr().getValueType(),
23080 :
LLD->getExtensionType(),
23086 CombineTo(TheSelect,
Load);
23090 CombineTo(
LHS.getNode(),
Load.getValue(0),
Load.getValue(1));
23091 CombineTo(
RHS.getNode(),
Load.getValue(0),
Load.getValue(1));
23115 if (CC ==
ISD::SETGT && TLI.hasAndNot(N2)) {
23132 auto *N2C = dyn_cast<ConstantSDNode>(N2.
getNode());
23133 if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue() - 1)) == 0)) {
23134 unsigned ShCt = XType.
getSizeInBits() - N2C->getAPIntValue().logBase2() - 1;
23135 if (!TLI.shouldAvoidTransformToShift(XType, ShCt)) {
23138 AddToWorklist(Shift.
getNode());
23140 if (XType.
bitsGT(AType)) {
23142 AddToWorklist(Shift.
getNode());
23146 Shift = DAG.
getNOT(
DL, Shift, AType);
23153 if (TLI.shouldAvoidTransformToShift(XType, ShCt))
23158 AddToWorklist(Shift.
getNode());
23160 if (XType.
bitsGT(AType)) {
23162 AddToWorklist(Shift.
getNode());
23166 Shift = DAG.
getNOT(
DL, Shift, AType);
23176 EVT VT =
N->getValueType(0);
23180 if (!TLI.isBinOp(BinOpc) || (N2.
getOpcode() != BinOpc))
23226 EVT VT =
N->getValueType(0);
23228 bool IsFree = IsFabs ? TLI.isFAbsFree(VT) : TLI.isFNegFree(VT);
23234 EVT IntVT =
Int.getValueType();
23248 SignMask = ~SignMask;
23254 SignMask = ~SignMask;
23259 AddToWorklist(
Int.getNode());
23267 SDValue DAGCombiner::convertSelectOfFPConstantsToLoadOffset(
23270 if (!TLI.reduceSelectOfFPConstantLoads(N0.
getValueType()))
23275 auto *TV = dyn_cast<ConstantFPSDNode>(N2);
23276 auto *FV = dyn_cast<ConstantFPSDNode>(N3);
23278 if (!TV || !FV || !TLI.isTypeLegal(VT))
23283 TLI.isFPImmLegal(TV->getValueAPF(), TV->getValueType(0), ForCodeSize) ||
23284 TLI.isFPImmLegal(FV->getValueAPF(), FV->getValueType(0), ForCodeSize))
23289 if (!TV->hasOneUse() && !FV->hasOneUse())
23293 const_cast<ConstantFP*
>(TV->getConstantFPValue()) };
23301 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
23310 AddToWorklist(
Cond.getNode());
23312 AddToWorklist(CstOffset.
getNode());
23314 AddToWorklist(CPIdx.
getNode());
23324 bool NotExtCompare) {
23326 if (N2 == N3)
return N2;
23331 auto *N1C = dyn_cast<ConstantSDNode>(N1.
getNode());
23332 auto *N2C = dyn_cast<ConstantSDNode>(N2.
getNode());
23333 auto *N3C = dyn_cast<ConstantSDNode>(N3.
getNode());
23337 AddToWorklist(
SCC.getNode());
23338 if (
auto *SCCC = dyn_cast<ConstantSDNode>(
SCC)) {
23341 return !(SCCC->isZero()) ? N2 : N3;
23346 convertSelectOfFPConstantsToLoadOffset(
DL, N0, N1, N2, N3, CC))
23349 if (
SDValue V = foldSelectCCToShiftAnd(
DL, N0, N1, N2, N3, CC))
23361 auto *ConstAndRHS = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
23362 if (ConstAndRHS && ConstAndRHS->getAPIntValue().countPopulation() == 1) {
23364 const APInt &AndMask = ConstAndRHS->getAPIntValue();
23366 if (!TLI.shouldAvoidTransformToShift(VT, ShCt)) {
23385 bool Fold = N2C &&
isNullConstant(N3) && N2C->getAPIntValue().isPowerOf2();
23386 bool Swap = N3C &&
isNullConstant(N2) && N3C->getAPIntValue().isPowerOf2();
23388 if ((Fold || Swap) &&
23389 TLI.getBooleanContents(CmpOpVT) ==
23391 (!LegalOperations || TLI.isOperationLegal(
ISD::SETCC, CmpOpVT))) {
23400 if (NotExtCompare && N2C->isOne())
23416 AddToWorklist(
SCC.getNode());
23417 AddToWorklist(Temp.
getNode());
23422 unsigned ShCt = N2C->getAPIntValue().logBase2();
23423 if (TLI.shouldAvoidTransformToShift(VT, ShCt))
23447 if (
auto *ValueOnZeroC = dyn_cast<ConstantSDNode>(ValueOnZero)) {
23454 (!LegalOperations || TLI.isOperationLegal(
ISD::CTTZ, VT)))
23461 (!LegalOperations || TLI.isOperationLegal(
ISD::CTLZ, VT)))
23469 if (!NotExtCompare && N1C && N2C && N3C &&
23470 N2C->getAPIntValue() == ~N3C->getAPIntValue() &&
23492 bool foldBooleans) {
23494 DagCombineInfo(DAG,
Level,
false,
this);
23495 return TLI.SimplifySetCC(VT, N0, N1,
Cond, foldBooleans, DagCombineInfo,
DL);
23509 if (
SDValue S = TLI.BuildSDIV(
N, DAG, LegalOperations, Built)) {
23530 if (
SDValue S = TLI.BuildSDIVPow2(
N,
C->getAPIntValue(), DAG, Built)) {
23550 if (
SDValue S = TLI.BuildUDIV(
N, DAG, LegalOperations, Built)) {
23583 EVT VT =
Op.getValueType();
23590 int Enabled = TLI.getRecipEstimateDivEnabled(VT, MF);
23591 if (
Enabled == TLI.ReciprocalEstimate::Disabled)
23596 int Iterations = TLI.getDivRefinementSteps(VT, MF);
23598 AddToWorklist(Est.getNode());
23606 for (
int i = 0; i < Iterations; ++i) {
23609 if (i == Iterations - 1) {
23611 AddToWorklist(MulEst.
getNode());
23615 AddToWorklist(NewEst.
getNode());
23618 (i == Iterations - 1 ?
N : FPOne), NewEst, Flags);
23619 AddToWorklist(NewEst.
getNode());
23622 AddToWorklist(NewEst.
getNode());
23625 AddToWorklist(Est.getNode());
23630 AddToWorklist(Est.getNode());
23646 unsigned Iterations,
23648 EVT VT =
Arg.getValueType();
23658 for (
unsigned i = 0; i < Iterations; ++i) {
23678 unsigned Iterations,
23680 EVT VT =
Arg.getValueType();
23691 for (
unsigned i = 0; i < Iterations; ++i) {
23700 if (Reciprocal || (i + 1) < Iterations) {
23723 EVT VT =
Op.getValueType();
23730 int Enabled = TLI.getRecipEstimateSqrtEnabled(VT, MF);
23731 if (
Enabled == TLI.ReciprocalEstimate::Disabled)
23736 int Iterations = TLI.getSqrtRefinementSteps(VT, MF);
23738 bool UseOneConstNR =
false;
23740 TLI.getSqrtEstimate(
Op, DAG,
Enabled, Iterations, UseOneConstNR,
23742 AddToWorklist(Est.
getNode());
23745 Est = UseOneConstNR
23746 ? buildSqrtNROneConst(
Op, Est, Iterations, Flags, Reciprocal)
23747 : buildSqrtNRTwoConst(
Op, Est, Iterations, Flags, Reciprocal);
23758 Test, TLI.getSqrtResultForDenormInput(
Op, DAG), Est);
23767 return buildSqrtEstimateImpl(
Op, Flags,
true);
23771 return buildSqrtEstimateImpl(
Op, Flags,
false);
23777 struct MemUseCharacteristics {
23786 auto getCharacteristics = [](
SDNode *
N) -> MemUseCharacteristics {
23787 if (
const auto *LSN = dyn_cast<LSBaseSDNode>(
N)) {
23789 if (
auto *
C = dyn_cast<ConstantSDNode>(LSN->getOffset()))
23791 ?
C->getSExtValue()
23793 ? -1 *
C->getSExtValue()
23797 return {LSN->isVolatile(), LSN->isAtomic(), LSN->getBasePtr(),
23800 LSN->getMemOperand()};
23802 if (
const auto *LN = cast<LifetimeSDNode>(
N))
23803 return {
false ,
false, LN->getOperand(1),
23804 (LN->hasOffset()) ? LN->getOffset() : 0,
23809 return {
false ,
false,
SDValue(),
23814 MemUseCharacteristics MUC0 = getCharacteristics(Op0),
23815 MUC1 = getCharacteristics(Op1);
23818 if (MUC0.BasePtr.getNode() && MUC0.BasePtr == MUC1.BasePtr &&
23819 MUC0.Offset == MUC1.Offset)
23823 if (MUC0.IsVolatile && MUC1.IsVolatile)
23828 if (MUC0.IsAtomic && MUC1.IsAtomic)
23831 if (MUC0.MMO && MUC1.MMO) {
23832 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) ||
23833 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore()))
23840 if (BaseIndexOffset::computeAliasing(Op0, MUC0.NumBytes, Op1, MUC1.NumBytes,
23846 if (!MUC0.MMO || !MUC1.MMO)
23852 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) ||
23853 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore()))
23861 int64_t SrcValOffset0 = MUC0.MMO->getOffset();
23862 int64_t SrcValOffset1 = MUC1.MMO->getOffset();
23863 Align OrigAlignment0 = MUC0.MMO->getBaseAlign();
23864 Align OrigAlignment1 = MUC1.MMO->getBaseAlign();
23865 auto &Size0 = MUC0.NumBytes;
23866 auto &Size1 = MUC1.NumBytes;
23867 if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
23868 Size0.hasValue() && Size1.hasValue() && *Size0 == *Size1 &&
23869 OrigAlignment0 > *Size0 && SrcValOffset0 % *Size0 == 0 &&
23870 SrcValOffset1 % *Size1 == 0) {
23871 int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.
value();
23872 int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.
value();
23876 if ((OffAlign0 + *Size0) <= OffAlign1 || (OffAlign1 + *Size1) <= OffAlign0)
23889 if (
UseAA && AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() &&
23890 Size0.hasValue() && Size1.hasValue()) {
23892 int64_t MinOffset =
std::min(SrcValOffset0, SrcValOffset1);
23893 int64_t Overlap0 = *Size0 + SrcValOffset0 - MinOffset;
23894 int64_t Overlap1 = *Size1 + SrcValOffset1 - MinOffset;
23909 void DAGCombiner::GatherAllAliases(
SDNode *
N,
SDValue OriginalChain,
23916 const bool IsLoad = isa<LoadSDNode>(
N) && cast<LoadSDNode>(
N)->isSimple();
23920 unsigned Depth = 0;
23924 switch (
C.getOpcode()) {
23933 bool IsOpLoad = isa<LoadSDNode>(
C.getNode()) &&
23934 cast<LSBaseSDNode>(
C.getNode())->isSimple();
23935 if ((IsLoad && IsOpLoad) || !
mayAlias(
N,
C.getNode())) {
23937 C =
C.getOperand(0);
23946 C =
C.getOperand(0);
23955 C =
C.getOperand(0);
23968 while (!Chains.
empty()) {
23981 if (
Depth > TLI.getGatherAllAliasesMaxDepth()) {
24002 if (ImproveChain(Chain)) {
24024 GatherAllAliases(
N, OldChain, Aliases);
24027 if (Aliases.
size() == 0)
24031 if (Aliases.
size() == 1)
24040 struct UnitT { } Unit;
24041 bool operator==(
const UnitT &,
const UnitT &) {
return true; }
24042 bool operator!=(
const UnitT &,
const UnitT &) {
return false; }
24058 bool DAGCombiner::parallelizeChainedStores(
StoreSDNode *St) {
24075 if (!
BasePtr.getBase().getNode())
24079 if (
BasePtr.getBase().isUndef())
24096 if (Chain->getMemoryVT().isScalableVector())
24103 if (!Chain->isSimple() || Chain->isIndexed())
24112 int64_t Length = (Chain->getMemoryVT().getSizeInBits() + 7) / 8;
24115 auto I = Intervals.find(
Offset);
24117 if (
I != Intervals.end() &&
I.start() < (
Offset + Length))
24120 if (
I != Intervals.begin() && (--
I).stop() <=
Offset)
24129 if (ChainedStores.
size() == 0)
24136 for (
unsigned I = ChainedStores.
size();
I;) {
24138 SDValue BetterChain = FindBetterChain(S, NewChain);
24142 ChainedStores[
I] = S;
24146 SDValue BetterChain = FindBetterChain(St, NewChain);
24161 auto hasImprovedChain = [&](
SDValue ST) ->
bool {
24162 return ST->getOperand(0) != NewChain;
24164 bool AddNewChain =
llvm::all_of(TFOps, hasImprovedChain);
24174 AddToWorklist(
Op.getNode());
24175 AddToWorklist(STChain);
24179 bool DAGCombiner::findBetterNeighborChains(
StoreSDNode *St) {
24186 if (!
BasePtr.getBase().getNode())
24190 if (
BasePtr.getBase().isUndef())
24194 if (parallelizeChainedStores(St))
24199 if (St->
getChain() != BetterChain) {
24200 replaceStoreChain(St, BetterChain);
24210 DAGCombiner(*
this, AA, OptLevel).Run(
Level);
static bool mayAlias(MachineInstr &MIa, SmallVectorImpl< MachineInstr * > &MemInsns, AliasAnalysis *AA)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static cl::opt< bool > UseAA("aarch64-use-aa", cl::init(true), cl::desc("Enable the use of AA during codegen."))
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static uint64_t * getMemory(unsigned numWords)
A utility function for allocating memory and checking for allocation failure.
This file implements a class to represent arbitrary precision integral constant values and operations...
This file contains the simple types necessary to represent the attributes associated with functions a...
SmallVector< MachineOperand, 4 > Cond
BlockVerifier::State From
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static bool splitMergedValStore(StoreInst &SI, const DataLayout &DL, const TargetLowering &TLI)
For the instruction sequence of store below, F and I values are bundled together as an i64 value befo...
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques=false)
static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0, unsigned ExtOpc, SmallVectorImpl< SDNode * > &ExtendNodes, const TargetLowering &TLI)
static bool CanCombineFCOPYSIGN_EXTEND_ROUND(SDNode *N)
copysign(x, fp_extend(y)) -> copysign(x, y) copysign(x, fp_round(y)) -> copysign(x,...
static cl::opt< bool > CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden, cl::desc("Enable DAG combiner's use of IR alias analysis"))
static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG)
static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG)
static SDValue scalarizeBinOpOfSplats(SDNode *N, SelectionDAG &DAG, const SDLoc &DL)
If a vector binop is performed on splat values, it may be profitable to extract, scalarize,...
static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift, SDValue ExtractFrom, SDValue &Mask, const SDLoc &DL)
Helper function for visitOR to extract the needed side of a rotate idiom from a shl/srl/mul/udiv.
static bool getCombineLoadStoreParts(SDNode *N, unsigned Inc, unsigned Dec, bool &IsLoad, bool &IsMasked, SDValue &Ptr, const TargetLowering &TLI)
static SDNode * getPostIndexedLoadStoreOp(SDNode *N, bool &IsLoad, bool &IsMasked, SDValue &Ptr, SDValue &BasePtr, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue tryToFoldExtOfMaskedLoad(SelectionDAG &DAG, const TargetLowering &TLI, EVT VT, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType, ISD::NodeType ExtOpc)
static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned, const TargetLowering &TLI)
Return true if divmod libcall is available.
static SDValue reduceBuildVecToShuffleWithZero(SDNode *BV, SelectionDAG &DAG)
static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1, SelectionDAG &DAG, const SDLoc &DL)
Given the operands of an add/sub operation, see if the 2nd operand is a masked 0/1 whose source opera...
static SDValue simplifyShuffleOfShuffle(ShuffleVectorSDNode *Shuf)
If we have a unary shuffle of a shuffle, see if it can be folded away completely.
static bool canSplitIdx(LoadSDNode *LD)
static SDValue ShrinkLoadReplaceStoreWithStore(const std::pair< unsigned, unsigned > &MaskInfo, SDValue IVal, StoreSDNode *St, DAGCombiner *DC)
Check to see if IVal is something that provides a value as specified by MaskInfo.
static ConstantSDNode * getAsNonOpaqueConstant(SDValue N)
If N is a ConstantSDNode with isOpaque() == false return it casted to a ConstantSDNode pointer else n...
static void adjustCostForPairing(SmallVectorImpl< LoadedSlice > &LoadedSlices, LoadedSlice::Cost &GlobalLSCost)
Adjust the GlobalLSCost according to the target paring capabilities and the layout of the slices.
static SDValue narrowInsertExtractVectorBinOp(SDNode *Extract, SelectionDAG &DAG, bool LegalOperations)
static bool matchRotateHalf(SelectionDAG &DAG, SDValue Op, SDValue &Shift, SDValue &Mask)
Match "(X shl/srl V1) & V2" where V2 may not be present.
static bool isCompatibleLoad(SDValue N, unsigned ExtOpcode)
Check if N satisfies: N is used once.
static SDValue partitionShuffleOfConcats(SDNode *N, SelectionDAG &DAG)
static SDValue narrowExtractedVectorBinOp(SDNode *Extract, SelectionDAG &DAG, bool LegalOperations)
If we are extracting a subvector produced by a wide binary operator try to use a narrow binary operat...
static bool areUsedBitsDense(const APInt &UsedBits)
Check that all bits set in UsedBits form a dense region, i.e., UsedBits looks like 0....
static SDValue getInputChainForNode(SDNode *N)
Given a node, return its input chain if it has one, otherwise return a null sd operand.
static Optional< bool > isBigEndian(const ArrayRef< int64_t > ByteOffsets, int64_t FirstOffset)
static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG)
If we are extracting a subvector from a wide vector load, convert to a narrow load to eliminate the e...
static ElementCount numVectorEltsOrZero(EVT T)
static SDValue foldSelectWithIdentityConstant(SDNode *N, SelectionDAG &DAG, bool ShouldCommuteOperands)
This inverts a canonicalization in IR that replaces a variable select arm with an identity constant.
static SDValue widenCtPop(SDNode *Extend, SelectionDAG &DAG)
Given an extending node with a pop-count operand, if the target does not support a pop-count in the n...
static const Optional< ByteProvider > calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth, bool Root=false)
Recursively traverses the expression calculating the origin of the requested byte of the given value.
static SDValue foldSelectOfConstantsUsingSra(SDNode *N, SelectionDAG &DAG)
If a (v)select has a condition value that is a sign-bit test, try to smear the condition operand sign...
static SDValue combineADDCARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG, SDValue X, SDValue Carry0, SDValue Carry1, SDNode *N)
If we are facing some sort of diamond carry propapagtion pattern try to break it up to generate somet...
static SDValue replaceShuffleOfInsert(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
If a shuffle inserts exactly one element from a source vector operand into another vector operand and...
static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner, const TargetLowering &TLI, EVT VT, bool LegalOperations, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType)
static cl::opt< std::string > CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden, cl::desc("Only use DAG-combiner alias analysis in this" " function"))
static bool isConstantSplatVectorMaskForType(SDNode *N, EVT ScalarTy)
static SDValue formSplatFromShuffles(ShuffleVectorSDNode *OuterShuf, SelectionDAG &DAG)
Combine shuffle of shuffle of the form: shuf (shuf X, undef, InnerMask), undef, OuterMask --> splat X...
static bool canFoldInAddressingMode(SDNode *N, SDNode *Use, SelectionDAG &DAG, const TargetLowering &TLI)
Return true if 'Use' is a load or a store that uses N as its base pointer and that N may be folded in...
static unsigned bigEndianByteAt(unsigned BW, unsigned i)
bool refineUniformBase(SDValue &BasePtr, SDValue &Index, SelectionDAG &DAG)
static SDValue foldExtractSubvectorFromShuffleVector(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalOperations)
Given EXTRACT_SUBVECTOR(VECTOR_SHUFFLE(Op0, Op1, Mask)), try to produce VECTOR_SHUFFLE(EXTRACT_SUBVEC...
static SDValue combineConcatVectorOfExtracts(SDNode *N, SelectionDAG &DAG)
static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG, bool LegalOperations)
Transform a vector binary operation into a scalar binary operation by moving the math/logic after an ...
static bool hasNoInfs(const TargetOptions &Options, SDValue N)
static SDValue isSaturatingMinMax(SDValue N0, SDValue N1, SDValue N2, SDValue N3, ISD::CondCode CC, unsigned &BW, bool &Unsigned)
static cl::opt< bool > MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true), cl::desc("DAG combiner may split indexing from loads"))
static SDValue combineShuffleToVectorExtend(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalOperations)
static SDValue PerformUMinFpToSatCombine(SDValue N0, SDValue N1, SDValue N2, SDValue N3, ISD::CondCode CC, SelectionDAG &DAG)
static SDValue tryToFoldExtendSelectLoad(SDNode *N, const TargetLowering &TLI, SelectionDAG &DAG)
Fold (sext (select c, load x, load y)) -> (select c, sextload x, sextload y) (zext (select c,...
static cl::opt< unsigned > StoreMergeDependenceLimit("combiner-store-merge-dependence-limit", cl::Hidden, cl::init(10), cl::desc("Limit the number of times for the same StoreNode and RootNode " "to bail out in store merging dependence check"))
static SDValue foldAndToUsubsat(SDNode *N, SelectionDAG &DAG)
For targets that support usubsat, match a bit-hack form of that operation that ends in 'and' and conv...
static SDValue stripTruncAndExt(SDValue Value)
static cl::opt< unsigned > TokenFactorInlineLimit("combiner-tokenfactor-inline-limit", cl::Hidden, cl::init(2048), cl::desc("Limit the number of operands to inline for Token Factors"))
static SDValue foldShuffleOfConcatUndefs(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
Try to convert a wide shuffle of concatenated vectors into 2 narrow shuffles followed by concatenatio...
static SDValue combineShuffleOfSplatVal(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
static auto getFirstIndexOf(R &&Range, const T &Val)
bool refineIndexType(MaskedGatherScatterSDNode *MGS, SDValue &Index, bool Scaled, SelectionDAG &DAG)
static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG)
Try to fold a 'not' shifted sign-bit with add/sub with constant operand into a shift and add with a d...
static int getShuffleMaskIndexOfOneElementFromOp0IntoOp1(ArrayRef< int > Mask)
If the shuffle mask is taking exactly one element from the first vector operand and passing through a...
static bool isContractableFMUL(const TargetOptions &Options, SDValue N)
static bool areSlicesNextToEachOther(const LoadedSlice &First, const LoadedSlice &Second)
Check whether or not First and Second are next to each other in memory.
static bool isBSwapHWordPair(SDValue N, MutableArrayRef< SDNode * > Parts)
static SDValue foldFPToIntToFP(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue getTruncatedUSUBSAT(EVT DstVT, EVT SrcVT, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &DL)
static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG)
static SDValue foldBoolSelectToLogic(SDNode *N, SelectionDAG &DAG)
static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS, SDValue RHS, const TargetLowering &TLI)
static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG, const TargetLowering &TLI, bool Force)
Flips a boolean if it is cheaper to compute.
static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op, KnownBits &Known)
static SDValue getSubVectorSrc(SDValue V, SDValue Index, EVT SubVT)
static SDValue getAsCarry(const TargetLowering &TLI, SDValue V)
static cl::opt< bool > StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden, cl::desc("Bypass the profitability model of load slicing"), cl::init(false))
Hidden option to stress test load slicing, i.e., when this option is enabled, load slicing bypasses m...
static SDValue combineShiftOfShiftedLogic(SDNode *Shift, SelectionDAG &DAG)
If we have a shift-by-constant of a bitwise logic op that itself has a shift-by-constant operand with...
static SDValue foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI, SDValue Carry0, SDValue Carry1, SDNode *N)
static std::pair< unsigned, unsigned > CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain)
Check to see if V is (and load (ptr), imm), where the load is having specific bytes cleared out.
static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset=0)
static SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode CC, const TargetLowering &TLI, SelectionDAG &DAG)
Generate Min/Max node.
static SDNode * getBuildPairElt(SDNode *N, unsigned i)
static SDValue combineShiftToMULH(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue PerformMinMaxFpToSatCombine(SDValue N0, SDValue N1, SDValue N2, SDValue N3, ISD::CondCode CC, SelectionDAG &DAG)
static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize, SelectionDAG &DAG, bool IsRotate)
static SDValue visitORCommutative(SelectionDAG &DAG, SDValue N0, SDValue N1, SDNode *N)
OR combines for which the commuted variant will be tried as well.
static cl::opt< bool > EnableShrinkLoadReplaceStoreWithStore("combiner-shrink-load-replace-store-with-store", cl::Hidden, cl::init(true), cl::desc("DAG combiner enable load/<replace bytes>/store with " "a narrower store"))
static bool shouldCombineToPostInc(SDNode *N, SDValue Ptr, SDNode *PtrUse, SDValue &BasePtr, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI, SelectionDAG &DAG, bool LegalTypes)
Try to fold a sext/zext/aext dag node into a ConstantSDNode or a build_vector of constants.
static SDValue foldExtendedSignBitTest(SDNode *N, SelectionDAG &DAG, bool LegalOperations)
static SDValue combineConcatVectorOfCasts(SDNode *N, SelectionDAG &DAG)
static SDValue combineShiftAnd1ToBitTest(SDNode *And, SelectionDAG &DAG)
Try to replace shift/logic that tests if a bit is clear with mask + setcc.
static SDValue matchBSwapHWordOrAndAnd(const TargetLowering &TLI, SelectionDAG &DAG, SDNode *N, SDValue N0, SDValue N1, EVT VT, EVT ShiftAmountTy)
static SDValue stripConstantMask(SelectionDAG &DAG, SDValue Op, SDValue &Mask)
static SDValue combineShuffleOfScalars(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG)
static SDValue foldVSelectToSignBitSplatMask(SDNode *N, SelectionDAG &DAG)
static SDValue combineConcatVectorOfConcatVectors(SDNode *N, SelectionDAG &DAG)
static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner, const TargetLowering &TLI, EVT VT, bool LegalOperations, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType, ISD::NodeType ExtOpc)
static cl::opt< bool > EnableStoreMerging("combiner-store-merging", cl::Hidden, cl::init(true), cl::desc("DAG combiner enable merging multiple stores " "into a wider store"))
static unsigned getPPCf128HiElementSelector(const SelectionDAG &DAG)
static cl::opt< bool > UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true), cl::desc("Enable DAG combiner's use of TBAA"))
static SDValue combineTruncationShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG)
static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT, SelectionDAG &DAG, bool LegalOperations)
static SDValue combineABSToABD(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
static bool isSlicingProfitable(SmallVectorImpl< LoadedSlice > &LoadedSlices, const APInt &UsedBits, bool ForCodeSize)
Check the profitability of all involved LoadedSlice.
static unsigned littleEndianByteAt(unsigned BW, unsigned i)
static bool isBSwapHWordElement(SDValue N, MutableArrayRef< SDNode * > Parts)
Return true if the specified node is an element that makes up a 32-bit packed halfword byteswap.
static SDValue FoldIntToFPToInt(SDNode *N, SelectionDAG &DAG)
static cl::opt< bool > EnableReduceLoadOpStoreWidth("combiner-reduce-load-op-store-width", cl::Hidden, cl::init(true), cl::desc("DAG combiner enable reducing the width of load/op/store " "sequence"))
static ManagedStatic< DebugCounter > DC
This file defines the DenseMap class.
Optional< std::vector< StOtherPiece > > Other
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
iv Induction Variable Users
This file implements a coalescing interval map for small objects.
static void removeFromWorklist(Instruction *I, std::vector< Instruction * > &Worklist)
Remove all instances of I from the worklist vector specified.
unsigned const TargetRegisterInfo * TRI
print Print MemDeps of function
This file provides utility analysis objects describing memory locations.
This file provides None, an enumerator for use in implicit constructors of various (usually templated...
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
This file provides Optional, a template class modeled in the spirit of OCaml's 'opt' variant.
const char LLVMTargetMachineRef LLVMPassBuilderOptionsRef Options
static cl::opt< bool > Aggressive("aggressive-ext-opt", cl::Hidden, cl::desc("Aggressive extension optimization"))
ManagedStatic< detail::RecordContext > Context
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSimple(Instruction *I)
This file implements a set that has insertion order iteration characteristics.
This file implements the SmallBitVector class.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
This file describes how to lower LLVM code to machine code.
static bool hasOneUse(unsigned Reg, MachineInstr *Def, MachineRegisterInfo &MRI, MachineDominatorTree &MDT, LiveIntervals &LIS)
static constexpr int Concat[]
const fltSemantics & getSemantics() const
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
APInt umul_ov(const APInt &RHS, bool &Overflow) const
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
static void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
bool isNegatedPowerOf2() const
Check if this APInt's negated value is a power of two greater than zero.
APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
bool isMinSignedValue() const
Determine if this is the smallest signed value.
uint64_t getZExtValue() const
Get zero extended value.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
APInt abs() const
Get the absolute value.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
bool isNegative() const
Determine sign of this APInt.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
int32_t exactLogBase2() const
APInt sextOrSelf(unsigned width) const
Sign extend or truncate to width.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
unsigned countLeadingZeros() const
The APInt version of the countLeadingZeros functions in MathExtras.h.
unsigned logBase2() const
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
bool getBoolValue() const
Convert APInt to a boolean value.
APInt smul_ov(const APInt &RHS, bool &Overflow) const
bool isMask(unsigned numBits) const
APInt sext(unsigned width) const
Sign extend to a new width.
APInt shl(unsigned shiftAmt) const
Left-shift function.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
APInt zextOrSelf(unsigned width) const
Zero extend or truncate to width.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
unsigned countTrailingOnes() const
Count the number of trailing one bits.
bool isOne() const
Determine if this is a value of 1.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
size_t size() const
size - Get the array size.
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
A "pseudo-class" with methods for operating on BUILD_VECTORs.
ISD::CondCode get() const
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
bool isNegative() const
Return true if the value is negative.
const APFloat & getValueAPF() const
bool isZero() const
Return true if the value is positive or negative zero.
ConstantFP - Floating Point Values [float, double].
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX)
uint64_t getZExtValue() const
int64_t getSExtValue() const
const APInt & getAPIntValue() const
const ConstantInt * getConstantIntValue() const
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
iterator find(const_arg_type_t< KeyT > Val)
bool erase(const KeyT &Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool isScalar() const
Counting predicates.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
AttributeList getAttributes() const
Return the attribute list for this Function.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
This class is used to form a handle around another node that is persistent and is updated across invo...
This is an important class for using LLVM in a threaded context.
Base class for LoadSDNode and StoreSDNode.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
bool isKnownMultipleOf(ScalarTy RHS) const
This function tells the caller whether the element count is known at compile time to be a multiple of...
LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
static ElementCount getFixed(ScalarTy MinVal)
This class is used to represent ISD::LOAD nodes.
const SDValue & getOffset() const
const SDValue & getBasePtr() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
static auto all_valuetypes()
SimpleValueType Iteration.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getIntegerVT(unsigned BitWidth)
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
This class is used to represent an MGATHER node.
const SDValue & getPassThru() const
ISD::LoadExtType getExtensionType() const
This is a base class used to represent MGATHER and MSCATTER nodes.
const SDValue & getScale() const
void setIndexType(ISD::MemIndexType IndexType)
bool isIndexScaled() const
const SDValue & getIndex() const
const SDValue & getBasePtr() const
ISD::MemIndexType getIndexType() const
How is Index applied to BasePtr when computing addresses.
const SDValue & getMask() const
This class is used to represent an MLOAD node.
bool isExpandingLoad() const
const SDValue & getBasePtr() const
const SDValue & getOffset() const
ISD::LoadExtType getExtensionType() const
const SDValue & getMask() const
const SDValue & getPassThru() const
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc,...
This class is used to represent an MSCATTER node.
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
This class is used to represent an MSTORE node.
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
const SDValue & getValue() const
const SDValue & getMask() const
const SDValue & getChain() const
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
const MachinePointerInfo & getPointerInfo() const
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
const SDValue & getBasePtr() const
bool isNonTemporal() const
bool isDereferenceable() const
unsigned getAlignment() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
static uint64_t getSizeOrUnknown(const TypeSize &T)
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
constexpr const T & getValue() const LLVM_LVALUE_FUNCTION
constexpr bool hasValue() const
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< value_op_iterator > op_values() const
SDNodeFlags getFlags() const
size_t use_size() const
Return the number of uses of this node.
void intersectFlagsWith(const SDNodeFlags Flags)
Clear any flags in this node that aren't also set in Flags.
TypeSize getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
SDVTList getVTList() const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
const APInt & getConstantOperandAPInt(unsigned Num) const
Helper method returns the APInt of a ConstantSDNode operand.
const SDValue & getOperand(unsigned Num) const
ArrayRef< SDUse > ops() const
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
iterator_range< use_iterator > uses()
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
void setFlags(SDNodeFlags NewFlags)
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth=2) const
Return true if this operand (which must be a chain) reaches the specified operand without crossing an...
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
virtual bool disableGenericCombines(CodeGenOpt::Level OptLevel) const
Help to insert SDNodeFlags automatically in transforming.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getSplatSourceVector(SDValue V, int &SplatIndex)
If V is a splatted value, return the source vector and its splat index.
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
bool isKnownNeverZero(SDValue Op) const
Test whether the given SDValue is known to contain non-zero value(s).
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl)
Constant fold a setcc to true or false.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDNode * isConstantIntBuildVectorOrConstantInt(SDValue N) const
Test whether the given value is a constant int or similar node.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=None, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
bool shouldOptForSize() const
OverflowKind computeOverflowKind(SDValue N0, SDValue N1) const
Determine if the result of the addition of 2 node can overflow.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
void DeleteNode(SDNode *N)
Remove the specified node from the system.
const TargetSubtargetInfo & getSubtarget() const
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal)
Try to simplify a select/vselect into 1 of its operands or a constant.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
SDNode * isConstantFPBuildVectorOrConstantFP(SDValue N) const
Test whether the given value is a constant FP or similar node.
SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits)
See if the specified operand can be simplified with the knowledge that only the bits specified by Dem...
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
bool isKnownToBeAPowerOfTwo(SDValue Val) const
Test if the given value is known to have exactly one bit set.
bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly=false, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
const DataLayout & getDataLayout() const
MachineFunction & getMachineFunction() const
SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
const TargetLowering & getTargetLoweringInfo() const
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
iterator_range< allnodes_iterator > allnodes()
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue is known to never be NaN.
SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
void Combine(CombineLevel Level, AAResults *AA, CodeGenOpt::Level OptLevel)
This iterates over the nodes in the SelectionDAG, folding certain types of nodes together,...
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, SDNodeFlags Flags)
Try to simplify a floating-point binary operation into 1 of its operands or a constant.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
const TargetLibraryInfo & getLibInfo() const
LLVMContext * getContext() const
bool isUndef(unsigned Opcode, ArrayRef< SDValue > Ops)
Return true if the result of this operation is always undefined.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
const TargetMachine & getTarget() const
DenormalMode getDenormalMode(EVT VT) const
Return the current function's default denormal handling kind for the given floating point type.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue simplifyShift(SDValue X, SDValue Y)
Try to simplify a shift into 1 of its operands or a constant.
void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits=0, unsigned SizeInBits=0, bool InvalidateDbg=true)
Transfer debug values from one node to another, while optionally generating fragment expressions for ...
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
A vector that has set insertion semantics.
bool remove(const value_type &X)
Remove an item from the set vector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
LLVM_NODISCARD T pop_back_val()
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
int getMaskElt(unsigned Idx) const
int getSplatIndex() const
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
ArrayRef< int > getMask() const
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
LLVM_NODISCARD bool empty() const
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
LLVM_NODISCARD bool empty() const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
LLVM_NODISCARD T pop_back_val()
void assign(size_type NumElts, ValueParamT Elt)
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
iterator insert(iterator I, T &&Elt)
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
bool has(LibFunc F) const
Tests whether a library function is available.
virtual bool isMulAddWithConstProfitable(const SDValue &AddNode, const SDValue &ConstNode) const
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
virtual bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
virtual bool hasAndNot(SDValue X) const
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
virtual bool enableAggressiveFMAFusion(EVT VT) const
Return true if target always benefits from combining into FMA for a given value type.
bool isIndexedStoreLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
SDValue promoteTargetBoolean(SelectionDAG &DAG, SDValue Bool, EVT ValVT) const
Promote the given target boolean to a target boolean of the given type.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual bool canCombineTruncStore(EVT ValVT, EVT MemVT, bool LegalOnly) const
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, bool *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
virtual bool convertSetCCLogicToBitwiseLogic(EVT VT) const
Use bitwise logic to make pairs of compares more efficient.
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool isVectorLoadExtDesirable(SDValue ExtVal) const
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
virtual bool hasBitPreservingFPLogic(EVT VT) const
Return true if it is safe to transform an integer-domain bitwise operation into the equivalent floati...
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
virtual bool shouldRemoveExtendFromGSIndex(EVT VT) const
virtual bool isFPExtFoldable(const MachineInstr &MI, unsigned Opcode, LLT DestTy, LLT SrcTy) const
Return true if an fpext operation input to an Opcode operation is free (for instance,...
virtual bool hasBitTest(SDValue X, SDValue Y) const
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
bool isTruncStoreLegal(EVT ValVT, EVT MemVT) const
Return true if the specified store with truncation is legal on this target.
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
virtual bool generateFMAsInMachineCombiner(EVT VT, CodeGenOpt::Level OptLevel) const
virtual bool isFPImmLegal(const APFloat &, EVT, bool ForCodeSize=false) const
Returns true if the target can instruction select the specified FP immediate natively.
virtual bool isExtractVecEltCheap(EVT VT, unsigned Index) const
Return true if extraction of a scalar element from the given vector type at the given index is cheap.
virtual bool shouldNormalizeToSelectSequence(LLVMContext &Context, EVT VT) const
Returns true if we should normalize select(N0&N1, X, Y) => select(N0, select(N1, X,...
bool isIndexedMaskedLoadLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
virtual bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
virtual bool isFsqrtCheap(SDValue X, SelectionDAG &DAG) const
Return true if SQRT(X) shouldn't be replaced with X*RSQRT(X).
virtual bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to fold a pair of shifts into a mask.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool shouldAvoidTransformToShift(EVT VT, unsigned Amount) const
Return true if creating a shift of the type by the given amount is not profitable.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
virtual bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal on this target.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual bool preferIncOfAddToSubOfNot(EVT VT) const
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
virtual bool isLegalAddImmediate(int64_t) const
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const
Return true if it is profitable to reduce a load to a smaller type.
virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const
virtual bool isFNegFree(EVT VT) const
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual bool isIntDivCheap(EVT VT, AttributeList Attr) const
Return true if integer divide is usually cheaper than a sequence of several shifts,...
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool mergeStoresAfterLegalization(EVT MemVT) const
Allow store merging for the specified type after legalization in addition to before legalization.
virtual bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const
Return true if it is cheaper to split the store of a merged int val from a pair of smaller values int...
bool isLoadExtLegalOrCustom(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal or custom on this target.
virtual bool storeOfVectorConstantIsCheap(EVT MemVT, unsigned NumElem, unsigned AddrSpace) const
Return true if it is expected to be cheaper to do a store of a non-zero vector constant with the give...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
virtual bool shouldFoldMaskToVariableShiftPair(SDValue X) const
There are two ways to clear extreme bits (either low or high): Mask: x & (-1 << y) (the instcombine c...
bool isIndexedLoadLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
virtual bool canMergeStoresTo(unsigned AS, EVT MemVT, const MachineFunction &MF) const
Returns if it's reasonable to merge stores to MemVT size.
bool isLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal on this target.
virtual bool shouldScalarizeBinop(SDValue VecOp) const
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: (store (y (conv x)), y*)) -> (store x,...
bool isIndexedMaskedStoreLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
bool hasTargetDAGCombine(ISD::NodeType NT) const
If true, the target has custom DAG combine transformations that it can perform for the specified node...
virtual bool shouldSplatInsEltVarIndex(EVT) const
Return true if inserting a scalar into a variable element of an undef vector is more efficiently hand...
NegatibleCost
Enum that specifies when a float negation is beneficial.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual bool isFMADLegal(const MachineInstr &MI, LLT Ty) const
Returns true if MI can be combined with another instruction to form TargetOpcode::G_FMAD.
virtual bool isNarrowingProfitable(EVT, EVT) const
Return true if it's profitable to narrow operations of type VT1 to VT2.
virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const
virtual bool isFAbsFree(EVT VT) const
Return true if an fabs operation is free to the point where it is never worthwhile to replace it with...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
virtual bool hasPairedLoad(EVT, Align &) const
Return true if the target supplies and combines to a paired load two loaded values of type LoadedType...
virtual bool convertSelectOfConstantsToMath(EVT VT) const
Return true if a select of constants (select Cond, C1, C2) should be transformed into simple math ops...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
virtual bool isReassocProfitable(SelectionDAG &DAG, SDValue N0, SDValue N1) const
SDValue getCheaperNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, unsigned Depth=0) const
This is the helper function to return the newly negated expression only when the cost is cheaper.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
virtual bool IsDesirableToPromoteOp(SDValue, EVT &) const
This method query the target whether it is beneficial for dag combiner to promote the specified node.
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is 'desirable' to us...
virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, NegatibleCost &Cost, unsigned Depth=0) const
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool isConstFalseVal(SDValue N) const
Return if the N is a constant or constant vector equal to the false value from getBooleanContents().
SDValue BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::UDIV node expressing a divide by constant, return a DAG expression to select that will ...
virtual bool getPostIndexedAddressParts(SDNode *, SDNode *, SDValue &, SDValue &, ISD::MemIndexedMode &, SelectionDAG &) const
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isConstTrueVal(SDValue N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
virtual bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to move this shift by a constant amount though its operand,...
virtual unsigned combineRepeatedFPDivisors() const
Indicate whether this target prefers to combine FDIVs with the same divisor.
virtual bool getPreIndexedAddressParts(SDNode *, SDValue &, SDValue &, ISD::MemIndexedMode &, SelectionDAG &) const
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::SDIV node expressing a divide by constant, return a DAG expression to select that will ...
virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SDIV lowering for power-of-2 denominators.
virtual bool isDesirableToTransformToIntegerOp(unsigned, EVT) const
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
unsigned NoSignedZerosFPMath
NoSignedZerosFPMath - This flag is enabled when the -enable-no-signed-zeros-fp-math is specified on t...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual bool useAA() const
Enable use of alias analysis during code generation (during MI scheduling, DAGCombine,...
ScalarTy getFixedSize() const
static TypeSize Fixed(ScalarTy MinVal)
The instances of the Type class are immutable: once they are created, they are never changed.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
This class is used to represent EVT's, which are used to parameterize some operations.
LLVM Value Representation.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Type * getType() const
All values are typed, get the type of this value.
iterator_range< use_iterator > uses()
int getNumOccurrences() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
const APInt & umax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be unsigned.
const APInt & umin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be unsigned.
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical AND between different comparisons of identical values: ((X op1 Y) & (X...
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ ADDCARRY
Carry-using nodes for multiple precision addition and subtraction.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on two values,...
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ LIFETIME_START
This corresponds to the llvm.lifetime.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ CARRY_FALSE
CARRY_FALSE - This node is used when folding other nodes, like ADDC/SUBC, which indicate the carry re...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BRCOND
BRCOND - Conditional branch.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool matchUnaryPredicate(SDValue Op, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant BUI...
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
Optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
bool isVPBinaryOp(unsigned Opcode)
Whether this is a vector-predicated binary operation opcode.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
Optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
bool allOperandsUndef(const SDNode *N)
Return true if the node has at least one operand and all operands of the specified node are ISD::UNDE...
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
bool matchBinaryPredicate(SDValue LHS, SDValue RHS, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
bool isVPReduction(unsigned Opcode)
Whether this is a vector-predicated reduction opcode.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical OR between different comparisons of identical values: ((X op1 Y) | (X ...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool match(Val *V, const Pattern &P)
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
@ Undef
Value of the register doesn't matter.
initializer< Ty > init(const Ty &Val)
DiagnosticInfoOptimizationBase::Argument NV
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
/file This file defines the SmallVector class.
std::string & operator+=(std::string &buffer, StringRef string)
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
bool operator<(int64_t V1, const APSInt &V2)
void stable_sort(R &&Range)
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
STATISTIC(NumFunctions, "Total number of functions")
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool operator==(uint64_t V1, const APInt &V2)
bool isConstantOrConstantVector(MachineInstr &MI, const MachineRegisterInfo &MRI)
Determines if MI defines a constant integer or a build vector of constant integers.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
llvm::SmallVector< int, 16 > createUnaryMask(ArrayRef< int > Mask, unsigned NumElts)
Given a shuffle mask for a binary shuffle, create the equivalent shuffle mask assuming both operands ...
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
bool getAlign(const Function &F, unsigned index, unsigned &align)
bool operator!=(uint64_t V1, const APInt &V2)
bool operator>=(int64_t V1, const APSInt &V2)
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
auto reverse(ContainerTy &&C, std::enable_if_t< has_rbegin< ContainerTy >::value > *=nullptr)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
unsigned M1(unsigned Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool operator>(int64_t V1, const APSInt &V2)
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
unsigned countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the most significant bit to the least stopping at the first 1.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
unsigned countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Align max(MaybeAlign Lhs, Align Rhs)
void sort(IteratorTy Start, IteratorTy End)
SDValue peekThroughOneUseBitcasts(SDValue V)
Return the non-bitcasted and one-use source operand of V if it exists.
bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
detail::ValueMatchesPoly< M > HasValue(M Matcher)
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
unsigned countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
bool isNullOrNullSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
bool is_splat(R &&Range)
Wrapper function around std::equal to detect if all elements in a container are same.
void narrowShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Replace each shuffle mask index with the scaled sequential indices for an equivalent mask of narrowed...
@ Or
Bitwise or logical OR of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
unsigned M0(unsigned Val)
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
std::enable_if_t<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type > cast(const Y &Val)
constexpr unsigned BitWidth
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
Align commonAlignment(Align A, Align B)
Returns the alignment that satisfies both alignments.
unsigned Log2(Align A)
Returns the log2 of the alignment.
bool operator<=(int64_t V1, const APSInt &V2)
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
int getSplatIndex(ArrayRef< int > Mask)
If all non-negative Mask elements are the same value, return that value.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
AAMDNodes concat(const AAMDNodes &Other) const
Determine the best AAMDNodes after concatenating two different locations together.
static constexpr roundingMode rmNearestTiesToEven
static unsigned int semanticsPrecision(const fltSemantics &)
opStatus
IEEE-754R 7: Default exception handling.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Represent subnormal handling kind for floating point instruction inputs and outputs.
static constexpr DenormalMode getIEEE()
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isRound() const
Return true if the size is a power-of-two number of bytes.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isZeroSized() const
Test if the given EVT has zero size, this will fail if called on a scalable type.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
Helper struct to store a base, index and offset that forms an address.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
bool isAllOnes() const
Returns true if value is all one bits.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoSignedZeros() const
bool hasApproximateFuncs() const
bool hasAllowReciprocal() const
bool hasAllowReassociation() const
void setNoUnsignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...
void AddToWorklist(SDNode *N)
bool recursivelyDeleteUnusedNodes(SDNode *N)
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...